From ab9a243d079d4e2122e094ae89d02d5ee7a4b791 Mon Sep 17 00:00:00 2001 From: Jan Macku Date: Thu, 3 Dec 2020 14:31:58 +0100 Subject: Rework of shell ci Bug fixes: * ci now checks only shell scripts New features: * add ability to ignore specific warnings * ci test files containing shebang #!/bin/bash or #!/bin/sh * ci test files with .sh extension * add ability to use inline comments in list files * use of colors in output * make code cleaner --- .ci/check-shell.sh | 60 ++++++++++++++++++++++----------- .ci/exception-list.txt | 4 +++ .ci/functions.sh | 91 +++++++++++++++++++++++++++++++++++++++++++++++--- .ci/script-list.txt | 2 ++ 4 files changed, 134 insertions(+), 23 deletions(-) create mode 100644 .ci/exception-list.txt diff --git a/.ci/check-shell.sh b/.ci/check-shell.sh index a4ea3c52..6fef7dd1 100755 --- a/.ci/check-shell.sh +++ b/.ci/check-shell.sh @@ -1,5 +1,6 @@ #!/bin/bash +# Path is wrong because of travis . ./.ci/functions.sh # ------------ # @@ -8,63 +9,84 @@ # https://medium.com/@joey_9999/how-to-only-lint-files-a-git-pull-request-modifies-3f02254ec5e0 # get names of files from PR (excluding deleted files) -# TRAVIS_COMMIT_RANGE - HEAD of destination branch ... HEAD of PR branch git diff --name-only --diff-filter=db "${TRAVIS_COMMIT_RANGE}" > ../pr-changes.txt # Find modified shell scripts -readarray list_of_changes < <(grep "^[^#]" ../pr-changes.txt) +list_of_changes=() +file_to_array "../pr-changes.txt" "list_of_changes" 0 +list_of_scripts=() +file_to_array "./.ci/script-list.txt" "list_of_scripts" 1 + +# Create list of scripts for testing list_of_changed_scripts=() for file in "${list_of_changes[@]}"; do - # https://stackoverflow.com/questions/19345872/how-to-remove-a-newline-from-a-string-in-bash - # Remove new line and add ./ at the beginning - is_it_script "$file" && list_of_changed_scripts+=("./${file//[$'\t\r\n ']}") + is_it_script "$file" "${list_of_scripts[@]}" && list_of_changed_scripts+=("./${file}") && continue + check_extension "$file" && list_of_changed_scripts+=("./${file}") && continue + check_shebang "$file" && list_of_changed_scripts+=("./${file}") done -echo "Changed shell scripts:" +# Get list of exceptions +list_of_exceptions=() +file_to_array "./.ci/exception-list.txt" "list_of_exceptions" 1 +string_of_exceptions=$(join_by , "${list_of_exceptions[@]}") + +echo -e "\n" +echo ":::::::::::::::::::::" +echo -e "::: ${WHITE}Shellcheck CI${NOCOLOR} :::" +echo ":::::::::::::::::::::" +echo -e "\n" + +echo -e "${WHITE}Changed shell scripts:${NOCOLOR}" echo "${list_of_changed_scripts[@]}" -echo "------------" +echo -e "${WHITE}List of shellcheck exceptions:${NOCOLOR}" +echo "${string_of_exceptions}" +echo -e "\n" # ------------ # # SHELLCHECK # # ------------ # # sed part is to edit shellcheck output so csdiff/csgrep knows it is shellcheck output (--format=gcc) -shellcheck --format=gcc "${list_of_changed_scripts[@]}" | sed -e 's|$| <--[shellcheck]|' > ../pr-br-shellcheck.err +shellcheck --format=gcc --exclude="${string_of_exceptions}" "${list_of_changed_scripts[@]}" 2> /dev/null | sed -e 's|$| <--[shellcheck]|' > ../pr-br-shellcheck.err # make destination branch -[[ ${TRAVIS_COMMIT_RANGE} =~ ^([0-9|a-f]*?)\. ]] && git checkout -b ci_br_dest "${BASH_REMATCH[1]}" +[[ ${TRAVIS_COMMIT_RANGE} =~ ^([0-9|a-f]*?)\. ]] && git checkout -q -b ci_br_dest "${BASH_REMATCH[1]}" -shellcheck --format=gcc "${list_of_changed_scripts[@]}" | sed -e 's|$| <--[shellcheck]|' > ../dest-br-shellcheck.err +shellcheck --format=gcc --exclude="${string_of_exceptions}" "${list_of_changed_scripts[@]}" 2> /dev/null | sed -e 's|$| <--[shellcheck]|' > ../dest-br-shellcheck.err # ------------ # # VALIDATION # # ------------ # exitstatus=0 +echo ":::::::::::::::::::::::::" +echo -e "::: ${WHITE}Validation Output${NOCOLOR} :::" +echo ":::::::::::::::::::::::::" +echo -e "\n" # Check output for Fixes csdiff --fixed "../dest-br-shellcheck.err" "../pr-br-shellcheck.err" > ../fixes.log if [ "$(cat ../fixes.log | wc -l)" -ne 0 ]; then - echo "Fixed bugs:" + echo -e "${GREEN}Fixed bugs:${NOCOLOR}" csgrep ../fixes.log - echo "------------" + echo "---------------------" else - echo "No Fixes!" - echo "------------" + echo -e "${YELLOW}No Fixes!${NOCOLOR}" + echo "---------------------" fi +echo -e "\n" # Check output for added bugs csdiff --fixed "../pr-br-shellcheck.err" "../dest-br-shellcheck.err" > ../bugs.log if [ "$(cat ../bugs.log | wc -l)" -ne 0 ]; then - echo "Added bugs, NEED INSPECTION:" + echo -e "${RED}Added bugs, NEED INSPECTION:${NOCOLOR}" csgrep ../bugs.log - echo "------------" + echo "---------------------" exitstatus=1 else - echo "No bugs added Yay!" - echo "------------" + echo -e "${GREEN}No bugs added Yay!${NOCOLOR}" + echo "---------------------" exitstatus=0 fi exit $exitstatus - diff --git a/.ci/exception-list.txt b/.ci/exception-list.txt new file mode 100644 index 00000000..5b093820 --- /dev/null +++ b/.ci/exception-list.txt @@ -0,0 +1,4 @@ +# List of Shelcheck codes which should be excluded from ci output +# Avoid spaces in list since they are counted as comment +SC1090 # Ignore when shellcheck can't follow non-constant source e.g. ``. "${path}"`` +SC2148 # Ignore missing shebang diff --git a/.ci/functions.sh b/.ci/functions.sh index 300f92df..cbe00874 100644 --- a/.ci/functions.sh +++ b/.ci/functions.sh @@ -1,9 +1,92 @@ # Function to check whether input param is on list of shell scripts # $1 - absolute path to file +# $@ - list of strings to compare with +# $? - return value - 0 when succes is_it_script () { - [ $# -eq 0 ] && return 1 - - readarray list_of_scripts < ./.ci/script-list.txt - echo "${list_of_scripts[@]}" | grep --silent "$1" && return 0 + [ $# -le 1 ] && return 1 + local file="$1" + shift + local scripts=("$@") + + [[ " ${scripts[*]} " =~ " ${file} " ]] && return 0 || return 2 +} + +# Function to check if given file has .sh extension +# https://stackoverflow.com/a/407229 +# $1 - absolute path to file +# $? - return value - 0 when succes +check_extension () { + [ $# -le 0 ] && return 1 + local file="$1" + + [ "${file: -3}" == ".sh" ] && return 0 || return 2 +} + +# Function to check if given file contain shell shebang (bash or sh) +# https://unix.stackexchange.com/a/406939 +# $1 - absolute path to file +# $? - return value - 0 when succes +check_shebang () { + [ $# -le 0 ] && return 1 + local file="$1" + + if IFS= read -r line < "./${file}" ; then + case $line in + "#!/bin/bash") return 0;; + "#!/bin/sh") return 0;; + *) return 1 + esac + fi +} + +# Function to prepare string from array of strings where first argument specify one character separator +# https://stackoverflow.com/a/17841619 +# $1 - Character used to join elements of array +# $@ - list of strings +# return value - string +join_by () { + local IFS="$1" + shift + echo "$*" +} + +# Function to get rid of comments represented by '#' +# $1 - file path +# $2 - name of variable where will be stored result array +# $3 - value 1|0 - does file content inline comments? +# $? - return value - 0 when succes +file_to_array () { + [ $# -le 2 ] && return 1 + local output=() + + [ "$3" -eq 0 ] && readarray output < <(grep -v "^#.*" "$1") # fetch array with lines from file while excluding '#' comments + [ "$3" -eq 1 ] && readarray output < <(cut -d ' ' -f 1 < <(grep -v "^#.*" "$1")) # fetch array with lines from file while excluding '#' comments + clean_array "$2" "${output[@]}" && return 0 +} + +# Function to get rid of spaces and new lines from array elements +# https://stackoverflow.com/a/9715377 +# https://stackoverflow.com/a/19347380 +# https://unix.stackexchange.com/a/225517 +# $1 - name of variable where will be stored result array +# $@ - source array +# $? - return value - 0 when succes +clean_array () { + [ $# -le 2 ] && return 1 + local output="$1" + shift + local input=("$@") + + for i in "${input[@]}"; do + eval $output+=\("${i//[$'\t\r\n ']}"\) + done } +# Color aliases use echo -e to use them +NOCOLOR='\033[0m' +RED='\033[0;31m' +GREEN='\033[0;32m' +ORANGE='\033[0;33m' +BLUE='\033[0;34m' +YELLOW='\033[1;33m' +WHITE='\033[1;37m' diff --git a/.ci/script-list.txt b/.ci/script-list.txt index 7323ceb7..053672df 100644 --- a/.ci/script-list.txt +++ b/.ci/script-list.txt @@ -1,4 +1,6 @@ # Tracker of all shell scripts in this repository +# Every new script should be added in order to test it with ci +# Avoid spaces in list since they are counted as comment etc/rc.d/init.d/functions etc/rc.d/init.d/network network-scripts/ifdown-bnep -- cgit v1.2.1