From c0a835b3605630543063ecf49cf58b6040ae87a9 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 8 Apr 2024 07:44:40 +0000 Subject: [PATCH 1/4] Added edge case where there is two kinds of error in one file --- .bin/validators.py | 129 +++++++++++++++++++++++---------------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/.bin/validators.py b/.bin/validators.py index 8f96395b..bf458ffc 100755 --- a/.bin/validators.py +++ b/.bin/validators.py @@ -158,10 +158,10 @@ for i in files: if event_type=="W": events["warn"].append([file,line_number]) - print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number) + #print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number) elif event_type=="E": events["error"].append([file,line_number]) - print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number) + #print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number) else: print_warn(i,"[!] Event decoding fail! Assuming checker %s is not wrapped calls compliant"%(i)) exec_status=2 @@ -217,39 +217,39 @@ for name,events,description in failed_checks: if file not in cleaned_failed_checks.keys(): cleaned_failed_checks[file]={} - cleaned_failed_checks[file]["warn"]=[] - cleaned_failed_checks[file]["error"]=[] - cleaned_failed_checks[file]["check"]=name - cleaned_failed_checks[file]["description"]=description + + if name not in cleaned_failed_checks[file].keys(): + cleaned_failed_checks[file][name]={} + cleaned_failed_checks[file][name]["warn"]=[] + cleaned_failed_checks[file][name]["error"]=[] + cleaned_failed_checks[file][name]["description"]=description - cleaned_failed_checks[file][err_type].append(int(line_number)) + cleaned_failed_checks[file][name][err_type].append(int(line_number)) -for file, warn_and_errors in cleaned_failed_checks.items(): +for file, checker in cleaned_failed_checks.items(): + for name, warn_and_errors in checker.items(): - warn=warn_and_errors["warn"] - error=warn_and_errors["error"] - - for k,v in warn_and_errors.items(): - - if k not in ["error","warn"]: - continue - - v.sort() - lines=[] - - for i in v: - i=int(i) - - if not lines: - lines.append([i,i]) + for k,v in warn_and_errors.items(): + + if k not in ["error","warn"]: continue - if lines[-1][1]+1==i: - lines[-1][1]=i - else: - lines.append([i,i]) + v.sort() + lines=[] - warn_and_errors[k]=lines + for i in v: + i=int(i) + + if not lines: + lines.append([i,i]) + continue + + if lines[-1][1]+1==i: + lines[-1][1]=i + else: + lines.append([i,i]) + + warn_and_errors[k]=lines if all_pass: error_text="All good! No checks failed." @@ -259,47 +259,48 @@ else: error_text=[] check_results={} - for file,warn_and_errors in cleaned_failed_checks.items(): - - error_msg="" - warn_msg="" - current_errors=[] - current_warnings=[] - checker_name=warn_and_errors["check"] - description=warn_and_errors["description"] - - if checker_name not in check_results.keys(): - check_results.update({checker_name:{"warn":[],"error":[]}}) - - for line_numbers in warn_and_errors["warn"]: + for file, checker in cleaned_failed_checks.items(): + for checker_name, warn_and_errors in checker.items(): - line_numbers[0]=str(line_numbers[0]) - line_numbers[1]=str(line_numbers[1]) + error_msg="" + warn_msg="" + current_errors=[] + current_warnings=[] - if line_numbers[0]==line_numbers[1]: - current_warnings.append(line_numbers[0]) - continue - - current_warnings.append('-'.join(line_numbers)) + if checker_name not in check_results.keys(): + check_results.update({checker_name:{"warn":[],"error":[],"description":warn_and_errors["description"]}}) - for line_numbers in warn_and_errors["error"]: + for line_numbers in warn_and_errors["warn"]: + + line_numbers[0]=str(line_numbers[0]) + line_numbers[1]=str(line_numbers[1]) - line_numbers[0]=str(line_numbers[0]) - line_numbers[1]=str(line_numbers[1]) + if line_numbers[0]==line_numbers[1]: + current_warnings.append(line_numbers[0]) + continue + + current_warnings.append('-'.join(line_numbers)) - if line_numbers[0]==line_numbers[1]: - current_errors.append(line_numbers[0]) - continue - - current_errors.append('-'.join(line_numbers)) + for line_numbers in warn_and_errors["error"]: - if current_errors: - error_msg=ERROR_MSG%(file,', '.join(current_errors)) - check_results[checker_name]["error"].append(error_msg) + line_numbers[0]=str(line_numbers[0]) + line_numbers[1]=str(line_numbers[1]) - if current_warnings: - warn_msg=WARN_MSG%(file,', '.join(current_warnings)) - check_results[checker_name]["warn"].append(warn_msg) + if line_numbers[0]==line_numbers[1]: + current_errors.append(line_numbers[0]) + continue + + current_errors.append('-'.join(line_numbers)) + + if current_errors: + error_msg=ERROR_MSG%(file,', '.join(current_errors)) + check_results[checker_name]["error"].append(error_msg) + + if current_warnings: + warn_msg=WARN_MSG%(file,', '.join(current_warnings)) + check_results[checker_name]["warn"].append(warn_msg) + + print(check_results) for checker,results in check_results.items(): @@ -312,7 +313,7 @@ else: warn_msg='\n'.join(results["warn"]) else: warn_msg="There are no warnings for this check!" - error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,description,warn_msg,error_msg)) + error_text.append(FORMATTED_OUTPUT_FORMAT%(checker,results["description"],warn_msg,error_msg)) error_text='\n- - -\n'.join(error_text) From a1a38ed6485d9d3e5210e0cae50ff60a09ccdfb7 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 8 Apr 2024 07:48:00 +0000 Subject: [PATCH 2/4] Added back the annoying print statements --- .bin/validators.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.bin/validators.py b/.bin/validators.py index bf458ffc..3f225fd4 100755 --- a/.bin/validators.py +++ b/.bin/validators.py @@ -158,10 +158,10 @@ for i in files: if event_type=="W": events["warn"].append([file,line_number]) - #print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number) + print_err(file,"[!] Checker %s got a warning for %s on line %s"%(i,file,line_number),line=line_number) elif event_type=="E": events["error"].append([file,line_number]) - #print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number) + print_err(file,"[!] Checker %s got a error for %s on line %s"%(i,file,line_number),line=line_number) else: print_warn(i,"[!] Event decoding fail! Assuming checker %s is not wrapped calls compliant"%(i)) exec_status=2 From 607736efd409bab8341d6d094acae907f630ab3a Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:42:25 +0000 Subject: [PATCH 3/4] Used env vars as input instead and removed debug line --- .bin/validators.py | 15 ++++++++++++--- .github/workflows/wordlist-validator.yml | 4 +++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/.bin/validators.py b/.bin/validators.py index 3f225fd4..28d504f4 100755 --- a/.bin/validators.py +++ b/.bin/validators.py @@ -12,7 +12,6 @@ import os,subprocess,sys -args=sys.argv[1] files=[] STEP_SUMMARY_LOCATION="summary.md" IS_RUNNING_AS_ACTIONS=False @@ -73,6 +72,18 @@ ERROR_MSG="Errors in file %s on lines %s" WARNING_STRING="::warning file=%s,line=%s,col=%s,endColumn=%s::%s" ERROR_STRING="::error file=%s,line=%s,col=%s,endColumn=%s::%s" +if "CHANGED_FILES" not in os.environ: + print("[!] CHANGED_FILES environment variable not found!") + print("[-] This error may occur if you are running this script in your own machine\n") + if len(sys.argv) < 2: + print("[!] No arguments set, exiting.") + exit(2) + + args=sys.argv[1] +else: + args=os.environ["CHANGED_FILES"] + + if "GITHUB_STEP_SUMMARY" not in os.environ: print("[!] GITHUB_STEP_SUMMARY not found in system environments!") print("[-] This error may occur if you are running this script in your own machine\n") @@ -300,8 +311,6 @@ else: warn_msg=WARN_MSG%(file,', '.join(current_warnings)) check_results[checker_name]["warn"].append(warn_msg) - print(check_results) - for checker,results in check_results.items(): if len(results["error"])>0: diff --git a/.github/workflows/wordlist-validator.yml b/.github/workflows/wordlist-validator.yml index a7ca903d..6bbab8bb 100644 --- a/.github/workflows/wordlist-validator.yml +++ b/.github/workflows/wordlist-validator.yml @@ -28,4 +28,6 @@ jobs: uses: tj-actions/changed-files@v34 - name: Analyze all added or modified files run: | - ./.bin/validators.py "${{ steps.changed-files.outputs.all_changed_files }}" + ./.bin/validators.py + env: + CHANGED_FILES: ${{ steps.changed-files.outputs.all_changed_files }} \ No newline at end of file From 1c9376a8ebcb016424d94b4b496ec6b03911fcf7 Mon Sep 17 00:00:00 2001 From: Mo Langning <133737702+molangning@users.noreply.github.com> Date: Mon, 8 Apr 2024 08:48:56 +0000 Subject: [PATCH 4/4] Verbose logging --- .bin/validators.py | 1 + 1 file changed, 1 insertion(+) diff --git a/.bin/validators.py b/.bin/validators.py index 28d504f4..120003ab 100755 --- a/.bin/validators.py +++ b/.bin/validators.py @@ -83,6 +83,7 @@ if "CHANGED_FILES" not in os.environ: else: args=os.environ["CHANGED_FILES"] +print(f"[+] Checking these files {args}") if "GITHUB_STEP_SUMMARY" not in os.environ: print("[!] GITHUB_STEP_SUMMARY not found in system environments!")