diff --git a/.bin/check-file-for-starting-slash b/.bin/check-file-for-starting-slash deleted file mode 100755 index ab951d03..00000000 --- a/.bin/check-file-for-starting-slash +++ /dev/null @@ -1,17 +0,0 @@ -#!/usr/bin/env bash -# Script to verify if a file contain any entries starting with a slash and show a warning into the github action result UI. -# Received as input a list of file paths to check separated by a space. -# More precisely the result of the "tj-actions/changed-files" github action. -## References: -# See https://github.com/tj-actions/changed-files -modified_files="$1" -for modified_file in $modified_files -do - echo "[+] Check $modified_file ..." - matches=$(grep -cE '^/[a-zA-Z0-9\._]+' $modified_file) - echo "Entries identified starting with a slash: $matches" - if [ $matches -ne 0 ] - then - echo "::warning file=$modified_file,line=1,col=1,endColumn=1::$matches entries start with a slash." - fi -done diff --git a/.bin/checkers/README.md b/.bin/checkers/README.md new file mode 100644 index 00000000..b7f6cd27 --- /dev/null +++ b/.bin/checkers/README.md @@ -0,0 +1,45 @@ +All the files here will be run by `validators.sh` as part of the check pipeline + +## Specifications +Scripts will be passed a space separated list of files to check +e.g `./script.sh "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt"` + +If you want your output to be parsed by the caller script, follow the below specs,else the output will be displayed raw in actions summaries + +## Wrapped calls + +### Wrapped calls checks + +- - - + +Scripts should check if its being run by the caller script. The environment variable `IS_RUNNING_UNDER_CALLER_SCRIPT` will be set to one + +### Wrapped calls Specifications + +- - - + +Checker scripts will now have to print out the name of the check followed by a new line + +Example `New line checker` + +This value will be used as the checker title + +After that, the descriptions will need to be printed out + +Example `To fix the error, you have to remove the empty lines or new lines at the end of the file` + +This is for contributors to understand why the checks failed + +- - - + +Outputs from now on will have to be in separate lines for each warnings or errors + +This is the format for errors + +Example `E,filename,line_number` + +In the above example, `E` stands for errors. Accepted values are `E` and `W` for errors and warnings respectively + +filename is the name of the file that the error comes from. + +line_number is the line the script detected the error in. \ No newline at end of file diff --git a/.bin/checkers/check-file-for-starting-slash.py b/.bin/checkers/check-file-for-starting-slash.py new file mode 100755 index 00000000..188a48e7 --- /dev/null +++ b/.bin/checkers/check-file-for-starting-slash.py @@ -0,0 +1,66 @@ +#!/usr/bin/python3 + +import os,sys + +if not sys.argv[1]: + exit(0) + +IS_WRAPPED=False + +if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ: + IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1" + +def print_normal(msg): + + if IS_WRAPPED: + return + print(msg) + +def print_err(file,line_number): + + if IS_WRAPPED: + print("E,%s,%s"%(file,line_number)) + +def print_warn(file,line_number): + + if IS_WRAPPED: + print("W,%s,%s"%(file,line_number)) + +print_normal("[+] Entries starting with slash checker") +if IS_WRAPPED: + print("Entries starting with slash check") + print("The warning are more for informative purposes and does not actually serve as a check. You can ignore this.") + +files=sys.argv[1].split(" ") + +for i in files: + if not os.path.isfile(i): + print_err(i,0) + print_normal("[!] %s does not exist!"%(i)) + exit(2) + +pass_status=True + +for i in files: + contents=open(i,"rb").read() + counter=1 + + for line in contents.splitlines(): + if line.startswith(b'/'): + print_normal("[!] Warning: %s starts with a slash on line %i!"%(i,counter)) + print_warn(i,counter) + pass_status=False + + counter+=1 + print_normal("[+] %s passed no starting slash check!"%(i)) + +if pass_status: + print_normal("[+] All files passed checks") + exit(0) + +print_normal("[!] Warning: One or more files failed to pass the checks") + +if IS_WRAPPED: + exit(0) +else: + exit(2) \ No newline at end of file diff --git a/.bin/checkers/new-line-and-empty-line-checker.py b/.bin/checkers/new-line-and-empty-line-checker.py new file mode 100755 index 00000000..b7d77cf0 --- /dev/null +++ b/.bin/checkers/new-line-and-empty-line-checker.py @@ -0,0 +1,88 @@ +#!/usr/bin/env python3 + +import os,sys + +if not sys.argv[1]: + exit(0) + +IS_WRAPPED=False + +if "IS_RUNNING_UNDER_CALLER_SCRIPT" in os.environ: + IS_WRAPPED=os.environ['IS_RUNNING_UNDER_CALLER_SCRIPT']=="1" + +def print_normal(msg): + + if IS_WRAPPED: + return + print(msg) + +def print_err(file,line_number): + + if IS_WRAPPED: + print("E,%s,%s"%(file,line_number)) + +def print_warn(file,line_number): + + if IS_WRAPPED: + print("W,%s,%s"%(file,line_number)) + +print_normal("[+] New line and empty line check") +if IS_WRAPPED: + print("New line and empty line check") + print("To fix the error, you would have to remove the empty lines or new lines at the end of the file.") + +files=sys.argv[1].split(" ") + +for i in files: + if not os.path.isfile(i): + print_err(i,0) + print_normal("[!] %s does not exist!"%(i)) + exit(2) + +overall_pass_status=True + +for i in files: + + contents=open(i,"rb").read() + line_number=len(contents.split(b'\n'))+1 + + if contents[-1:] == b'\n': + print_normal("[!] Warning: %s ends with a new line!"%(i)) + print_warn(i,line_number) + overall_pass_status=False + else: + print_normal("[+] %s passed new line check!"%(i)) + + counter=1 + + line_pass_status=True + + for line in contents.splitlines(): + if not line: + print_normal("[!] Warning: %s has an empty entry on line %i!"%(i,counter)) + print_warn(i,counter) + pass_status=False + line_pass_status=False + continue + + elif not line.strip(): + print_normal("[!] Warning: %s has an whitespace only entry on line %i!"%(i,counter)) + print_warn(i,counter) + pass_status=False + line_pass_status=False + continue + + counter+=1 + if line_pass_status: + print_normal("[+] %s passed empty line check!"%(i)) + +if overall_pass_status: + print_normal("[+] All files passed checks") + exit(0) + +print_normal("[!] Warning: One or more files failed to pass the checks") + +if IS_WRAPPED: + exit(0) +else: + exit(2) diff --git a/.bin/new-line-and-empty-line-checker.py b/.bin/new-line-and-empty-line-checker.py deleted file mode 100755 index 26cbb1d7..00000000 --- a/.bin/new-line-and-empty-line-checker.py +++ /dev/null @@ -1,38 +0,0 @@ -#!/usr/bin/env python3 - -# test string: ./.bin/new-line-checker.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt" - -import os -import sys - -print("[+] New line check") - -if not sys.argv[1]: - exit(0) - -files=sys.argv[1].split(" ") - -for i in files: - if not os.path.isfile(i): - print("[!] %s does not exist!"%(i)) - exit(2) - -for i in files: - contents=open(i,"rb").read() - - if contents[-1] == b'\n': - print("[!] %s ends with a new line!"%(i)) - exit(2) - print("[+] %s passed new line check!"%(i)) - - counter=1 - - for line in contents.splitlines(False): - if len(line)==0: - print("[!] %s has an empty entry at line %i!"%(i,counter)) - exit(2) - counter+=1 - print("[+] %s passed empty line check!"%(i)) - -print("[+] All files passed checks") -# exit(0) diff --git a/.bin/validators.py b/.bin/validators.py new file mode 100755 index 00000000..67b58f56 --- /dev/null +++ b/.bin/validators.py @@ -0,0 +1,323 @@ +#!/usr/bin/python3 + +# The workflow commands are from https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions +# TL;DR You can echo text out and the text can trigger workflow commands if in format "::workflow-command parameter1={data},parameter2={data}::{command value}" +# We are using the warnings and errors settings. +# You may use the below lines for testing +# +# Invalid test args ./.bin/validators.py "Fuzzing/file-extensions-all-cases.txt Fuzzing/file-extensions-lower-case.txt Fuzzing/file-extensions-upper-case.txt Fuzzing/file-extensions.txt" +# valid test args: ./.bin/validators.py "Discovery/Web-Content/trickest-robots-disallowed-wordlists/top-100.txt" +# +# Script by @molangning. I am the guy you want to ping when this script get borked, not the repository owners + +import os,subprocess,sys + +args=sys.argv[1] +files=[] +STEP_SUMMARY_LOCATION="summary.md" +IS_RUNNING_AS_ACTIONS=False +CHECK_MARK_EMOJI=":white_check_mark:" +CROSS_MARK_EMOJI=":negative_squared_cross_mark:" +QUESTION_MARK_EMOJI=":question:" +FORMATTED_OUTPUT_FORMAT=""" +### %s + +%s + +- - - + +Errors + +``` +%s +``` + +Warnings + +``` +%s +``` +""" +UNFORMATTED_OUTPUT_FORMAT=""" +### %s + +- - - + +Output + +``` +%s +``` +""" +SUMMARY_FORMAT=""" +# Checks summary + +This is a summary of the checks ran on the pushed files. If there is any errors please fix them before pushing again. +## Check statuses + +| Test name | Passed? | +| --------- | :-----: | +%s + +## Errors and warnings +%s + +## Other unformatted outputs +This section is for scripts that doesn't follow specifications + +%s +""" +TABLE_FORMAT="| %s | %s |\n" +WARN_MSG="Warnings in file %s on lines %s" +ERROR_MSG="Errors in file %s on lines %s" +WARNING_STRING="::warning file=%s,line=%s,col=%s,endColumn=%s::%s" +ERROR_STRING="::error file=%s,line=%s,col=%s,endColumn=%s::%s" + +if "GITHUB_STEP_SUMMARY" not in os.environ: + print("[!] GITHUB_STEP_SUMMARY not found in system environments!") + print("[-] This error may occur if you are running this script in your own machine\n") +else: + STEP_SUMMARY_LOCATION=os.environ["GITHUB_STEP_SUMMARY"] + IS_RUNNING_AS_ACTIONS=True + +if IS_RUNNING_AS_ACTIONS: + print("[+] Now running as github actions\n") +else: + print("[+] Now running as normal user\n") + +def print_warn(file,msg,line=1,col=1,endcol=1): + if IS_RUNNING_AS_ACTIONS: + print(WARNING_STRING%(file,line,col,endcol,msg)) + else: + print(msg) + + +def print_err(file,msg,line=1,col=1,endcol=1): + if IS_RUNNING_AS_ACTIONS: + print(ERROR_STRING%(file,line,col,endcol,msg)) + else: + print(msg) + +for root,_,file_list in os.walk('.bin/checkers'): + for file in file_list: + if file.endswith('.md'): + continue + + files.append(os.path.join(root,file)) + +sys_env=os.environ.copy() +sys_env.update({"IS_RUNNING_UNDER_CALLER_SCRIPT":"1"}) +all_events=[] + +for i in files: + events={"error":[],"warn":[],"raw_output":""} + is_using_wrapped_syntax=True + exec_status=1 # 0 for fail, 1 for success, 2 for unknown + description="" + + try: + output=subprocess.check_output([i,args],env=sys_env) + + except PermissionError: + print_warn(i,"[!] Not running test %s due to insufficient privileges!"%(i)) + print_warn(i,"[!] chmod +x it to let the workflow run the check.\n") + continue + + except subprocess.CalledProcessError as exec_err: + print_err(i,"[!] Error! Calling command with args %s failed!"%(str([i,args]))) + print_err(i,"[!] Process exited with error code %s and error message %s\n"%(exec_err.returncode,exec_err.output)) + continue + + output=output.decode('utf-8') + events["raw_output"]=output + + split_output=output.split('\n') + + if split_output[-1]=="": + split_output=split_output[:-1] + + if len(split_output)<2: + print_warn(i,"[!] Checker printed out less than two lines! Assuming not wrapped calls compliant.") + exec_status=2 + is_using_wrapped_syntax=False + + else: + + for line in split_output[2:]: + + if not len(line): + continue + + try: + event_type,file,line_number=line.split(',') + except: + print_warn(i,"[!] Split fail! Assuming checker %s is not wrapped calls compliant."%(i)) + is_using_wrapped_syntax=False + exec_status=2 + break + + if event_type=="W": + events["warn"].append([file,line_number]) + print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number) + elif event_type=="E": + events["error"].append([file,line_number]) + print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number) + else: + print_warn(i,"[!] Event decoding fail! Assuming checker %s is not wrapped calls compliant"%(i)) + exec_status=2 + break + + if is_using_wrapped_syntax: + script_name=split_output[0] + description=split_output[1] + if len(events["error"])==0 and len(events["warn"])==0: + print("[+] Ran %s and got no warnings or errors"%(script_name)) + exec_status=1 + else: + print("[+] Ran %s, got %i errors and %i warnings"%(script_name,len(events["error"]),len(events["warn"]))) + exec_status=0 + else: + print("[+] Ran checker %s but finished with unknown status"%(i)) + script_name=i + + all_events.append([script_name,events,exec_status,description]) + +all_pass=True +failed_checks=[] +unformatted_raw_output=[] +table_content="" +errors_encountered="" + +for name,events,exec_status,description in all_events: + + if exec_status==0: + table_content+=TABLE_FORMAT%(name,"No %s"%(CROSS_MARK_EMOJI)) + all_pass=False + failed_checks.append([name,events,description]) + elif exec_status==1: + table_content+=TABLE_FORMAT%(name,"Yes %s"%(CHECK_MARK_EMOJI)) + elif exec_status==2: + table_content+=TABLE_FORMAT%(name,"Unknown %s"%(QUESTION_MARK_EMOJI)) + unformatted_raw_output.append([name,events["raw_output"]]) + +formatted_raw_output=[] +for name,output in unformatted_raw_output: + formatted_raw_output.append(UNFORMATTED_OUTPUT_FORMAT%(name,output)) + +formatted_raw_output='\n- - -\n'.join(formatted_raw_output) +cleaned_failed_checks={} + +for name,events,description in failed_checks: + + for err_type,err in events.items(): + if err_type not in ["error","warn"]: + continue + + for file,line_number in err: + + if file not in cleaned_failed_checks.keys(): + cleaned_failed_checks[file]={} + cleaned_failed_checks[file]["warn"]=[] + cleaned_failed_checks[file]["error"]=[] + cleaned_failed_checks[file]["check"]=name + cleaned_failed_checks[file]["description"]=description + + cleaned_failed_checks[file][err_type].append(int(line_number)) + +for file, warn_and_errors in cleaned_failed_checks.items(): + + warn=warn_and_errors["warn"] + error=warn_and_errors["error"] + + for k,v in warn_and_errors.items(): + + if k not in ["error","warn"]: + continue + + v.sort() + lines=[] + + for i in v: + i=int(i) + + if not lines: + lines.append([i,i]) + continue + + if lines[-1][1]+1==i: + lines[-1][1]=i + else: + lines.append([i,i]) + + warn_and_errors[k]=lines + +if all_pass: + error_text="All good! No checks failed." + +else: + + error_text=[] + check_results={} + + for file,warn_and_errors in cleaned_failed_checks.items(): + + error_msg="" + warn_msg="" + current_errors=[] + current_warnings=[] + checker_name=warn_and_errors["check"] + description=warn_and_errors["description"] + + if checker_name not in check_results.keys(): + check_results.update({checker_name:{"warn":[],"error":[]}}) + + for line_numbers in warn_and_errors["warn"]: + + line_numbers[0]=str(line_numbers[0]) + line_numbers[1]=str(line_numbers[1]) + + if line_numbers[0]==line_numbers[1]: + current_warnings.append(line_numbers[0]) + continue + + current_warnings.append('-'.join(line_numbers)) + + for line_numbers in warn_and_errors["error"]: + + line_numbers[0]=str(line_numbers[0]) + line_numbers[1]=str(line_numbers[1]) + + if line_numbers[0]==line_numbers[1]: + current_errors.append(line_numbers[0]) + continue + + current_errors.append('-'.join(line_numbers)) + + if current_errors: + error_msg=ERROR_MSG%(file,', '.join(current_errors)) + check_results[checker_name]["error"].append(error_msg) + + if current_warnings: + warn_msg=WARN_MSG%(file,', '.join(current_warnings)) + check_results[checker_name]["warn"].append(warn_msg) + + for checker,results in check_results.items(): + + if len(results["error"])>0: + error_msg='\n'.join(results["error"]) + else: + error_msg="There are no errors for this check!" + + if len(results["warn"])>0: + warn_msg='\n'.join(results["warn"]) + else: + warn_msg="There are no warnings for this check!" + error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,description,warn_msg,error_msg)) + + error_text='\n- - -\n'.join(error_text) + +open(STEP_SUMMARY_LOCATION,"w").write(SUMMARY_FORMAT%(table_content,error_text,formatted_raw_output)) + +if not all_pass: + print_err(".bin/validators.py","[!] Not all checks passed.") + exit(2) \ No newline at end of file diff --git a/.bin/validators.sh b/.bin/validators.sh deleted file mode 100755 index bf2e3f18..00000000 --- a/.bin/validators.sh +++ /dev/null @@ -1,11 +0,0 @@ -#!/usr/bin/env bash - -# https://stackoverflow.com/questions/3822621/how-to-exit-if-a-command-failed - -set -e -set -o pipefail - -# wrapper for all the checking scripts -echo $1 -./.bin/check-file-for-starting-slash "$1" -./.bin/new-line-and-empty-line-checker.py "$1" \ No newline at end of file diff --git a/.github/workflows/wordlist-validator.yml b/.github/workflows/wordlist-validator.yml index 80a94422..a7ca903d 100644 --- a/.github/workflows/wordlist-validator.yml +++ b/.github/workflows/wordlist-validator.yml @@ -28,4 +28,4 @@ jobs: uses: tj-actions/changed-files@v34 - name: Analyze all added or modified files run: | - ./.bin/validators.sh "${{ steps.changed-files.outputs.all_changed_files }}" + ./.bin/validators.py "${{ steps.changed-files.outputs.all_changed_files }}"