COVERAGE_FILE_PATH = "./coverage_fe/index.html" JSON_COVERAGE_FILE_PATH = "./coverage_fe/coverage-final.json" THRESHOLD = 0.001 REPO_URL = "https://api.github.com/repos/Farmbot/Farmbot-Web-App" LATEST_COV_URL = "https://coveralls.io/github/FarmBot/Farmbot-Web-App.json" COV_API_BUILDS_PER_PAGE = 5 COV_BUILDS_TO_FETCH = 20 PULL_REQUEST = ENV.fetch("CIRCLE_PULL_REQUEST", "/0") CURRENT_BRANCH = ENV.fetch("CIRCLE_BRANCH", "staging") # "staging" or "pull/11" BASE_BRANCHES = ["master", "staging"] CURRENT_COMMIT = ENV.fetch("CIRCLE_SHA1", "") CSS_SELECTOR = ".fraction" FRACTION_DELIM = "/" # Fetch JSON over HTTP. Rails probably already has a helper for this :shrug: def open_json(url) begin JSON.parse(URI.open(url).read) rescue *[OpenURI::HTTPError, SocketError] => exception puts exception.message return {} end end # Don't fail on staging builds (i.e., not a pull request) to allow auto-deploys. def exit_0? CURRENT_BRANCH == "staging" end # Get pull request number. Return 0 if not a PR. def pr_number PULL_REQUEST.split("/")[-1].to_i end # Get pull request information from the GitHub API. def fetch_pull_data() if pr_number != 0 return open_json("#{REPO_URL}/pulls/#{pr_number}") end return {} end # Determine the base branch of the current build. def get_base_branch(pull_data) current_branch = BASE_BRANCHES.empty? || BASE_BRANCHES.include?(CURRENT_BRANCH) ? CURRENT_BRANCH : "staging" provided_base_branch = CURRENT_BRANCH.start_with?("master-hotfix/") ? "master" : nil; pull_data.dig("base", "ref") || provided_base_branch || current_branch end # Gather relevant coverage data. def relevant_data(build) { branch: build["branch"], commit: build["commit_sha"], percent: build["covered_percent"] } end # Fetch relevant coverage build data from commit. def fetch_build_data_from_commit(commit) if commit.nil? puts "Commit not found." build_data = {} else build_data = open_json("https://coveralls.io/builds/#{commit}.json") end return relevant_data(build_data) end # Fetch relevant remote coverage data for the latest commit on a branch. def fetch_latest_branch_build(branch) github_data = open_json("#{REPO_URL}/git/refs/heads/#{branch}") if github_data.is_a? Array github_data = {} # Didn't match a branch end commit = github_data.dig("object", "sha") return fetch_build_data_from_commit(commit) end # Fetch latest remote coverage data for a branch (commit fetched via GH PR API). def fetch_latest_pr_base_branch_build(branch) github_data = open_json("#{REPO_URL}/pulls?state=closed&base=#{branch}") commit = (github_data[0] || {}).dig("base", "sha") return fetch_build_data_from_commit(commit) end # Fetch a page of build coverage report results. def fetch_builds_for_page(page_number) open_json("#{LATEST_COV_URL}?page=#{page_number}")["builds"] || [] end # Number of coverage build data pages required to fetch the desired build count. def cov_pages_required (COV_BUILDS_TO_FETCH / COV_API_BUILDS_PER_PAGE.to_f).ceil end # Fetch coverage data from the last COV_BUILDS_TO_FETCH builds. def fetch_build_data() build_data = fetch_builds_for_page(1) for page_number in 2..cov_pages_required build_data.push(*fetch_builds_for_page(page_number)) end clean_build_data = build_data .reject { |build| build["covered_percent"].nil? } .reject { |build| build["branch"].include? "/" } puts "Using data from #{clean_build_data.length} of #{build_data.length}" \ " recent coverage builds." clean_build_data.map { |build| relevant_data(build) } end # Print history and return the most recent match for the provided branch. def latest_build_data(build_history, branch) if branch == "*" branch_builds = build_history else branch_builds = build_history.select { |build| build[:branch] == branch } end if branch_builds.length > 0 puts "\nCoverage history (newest to oldest):" branch_builds.map { |build| puts "#{build[:branch]}: #{build[:percent].round(3)}%" } branch_builds[0] else { branch: branch, commit: nil, percent: nil } end end # Calculate coverage results from JSON coverage report. def get_json_coverage_results() results = {lines: {covered: 0, total: 0}, branches: {covered: 0, total: 0}} begin data = open_json(JSON_COVERAGE_FILE_PATH) rescue Errno::ENOENT return results end data.each do |filename, file_coverage| lineMap = {} file_coverage["s"].each do |statement, count| line = file_coverage["statementMap"][statement]["start"]["line"] if lineMap[line].nil? || lineMap[line] < count lineMap[line] = count end end results[:lines][:covered] += lineMap.map{ |line, count| count } .filter{ |count| count != 0}.length results[:lines][:total] += lineMap.length branches = [] file_coverage["b"].each do |branch, counts| counts.map{ |count| branches.push(count) } end results[:branches][:covered] += branches.filter{ |count| count != 0 }.length results[:branches][:total] += branches.length end results end # on : def branch_info_string?(target, pull_data) unless pull_data.dig(target, "sha").nil? "#{pull_data.dig(target, "sha")} on #{pull_data.dig(target, "label")}" end end # Print a coverage difference summary string. def print_summary_text(build_percent, remote, pull_data) diff = (build_percent - remote[:percent]).round(3) direction = diff > 0 ? "increased" : "decreased" description = diff == 0 ? "remained the same at" : "#{direction} (#{diff}%) to" puts "Coverage #{description} #{build_percent.round(3)}%" \ " when pulling #{branch_info_string?("head", pull_data)}" \ " into #{branch_info_string?("base", pull_data) || remote[:branch]}." end def to_percent(pair) return ((pair.head / pair.tail) * 100).round(4) end namespace :coverage do desc "Verify code test coverage changes remain within acceptable thresholds." \ "Compares current test coverage percentage from Jest output to previous" \ "values from the base branch of a PR (or the build branch if not a PR)." \ "This task is used during ci to fail PR builds if test coverage" \ "decreases significantly and can also be run locally after running" \ "`jest --coverage` or `npm test-slow`." \ "The Coveralls stats reporter used to perform this check, but didn't" \ "compare against a PR's base branch and would always return 0% change." task run: :environment do begin # Fetch current build coverage data from the HTML summary. statements, branches, functions, lines = Nokogiri::HTML(URI.open(COVERAGE_FILE_PATH)) .css(CSS_SELECTOR) .map(&:text) .map { |x| x.split(FRACTION_DELIM).map(&:to_f) } .map { |x| Pair.new(*x) } rescue Errno::ENOENT end puts "\nUnable to determine coverage from HTML report." if lines.nil? puts "Checking JSON report..." if lines.nil? results = get_json_coverage_results() lines_json_report = Pair.new( results[:lines][:covered].to_f, results[:lines][:total].to_f) branches_json_report = Pair.new( results[:branches][:covered].to_f, results[:branches][:total].to_f) if results[:lines][:total] > 0 lines = lines || lines_json_report branches = branches || branches_json_report end covered = lines_json_report.head + branches_json_report.head total = lines_json_report.tail + branches_json_report.tail puts "JSON report aggregate: #{covered / total * 100}%" puts fallback_fraction = Pair.new(0.0, 1.0) puts "\nUnable to determine coverage from build." if lines.nil? statements = statements || fallback_fraction branches = branches || fallback_fraction functions = functions || fallback_fraction lines = lines || fallback_fraction puts puts "This build: #{CURRENT_COMMIT}" puts "Statements: #{to_percent(statements)}%" puts "Branches: #{to_percent(branches)}%" puts "Functions: #{to_percent(functions)}%" puts "Lines: #{to_percent(lines)}%" # Calculate an aggregate coverage percentage for the current build. covered = lines.head + branches.head total = lines.tail + branches.tail build_percent = (covered / total) * 100 puts "Aggregate: #{build_percent.round(4)}%" puts # Fetch remote build coverage data for the current branch. pull_request_data = fetch_pull_data() coverage_history_data = fetch_build_data() # Use fetched data. base_branch = get_base_branch(pull_request_data) remote = latest_build_data(coverage_history_data, base_branch) if remote[:percent].nil? puts "Coveralls data for '#{base_branch}' not found within history." puts "Attempting to get coveralls build data for latest commit." remote = fetch_latest_branch_build(base_branch) end if remote[:percent].nil? puts "Coverage data for latest '#{base_branch}' commit not available." puts "Attempting to use data from the previous commit (latest PR base)." remote = fetch_latest_pr_base_branch_build(base_branch) end if remote[:percent].nil? && base_branch != "staging" puts "Error getting coveralls data for '#{base_branch}'." puts "Attempting to use staging build coveralls data from history." remote = latest_build_data(coverage_history_data, "staging") end if remote[:percent].nil? puts "Error getting coveralls data for staging." puts "Attempting to use latest build coveralls data in history." remote = latest_build_data(coverage_history_data, "*") end if remote[:percent].nil? puts "Error getting coveralls data." puts "Using 100 instead of nil for remote coverage value." remote = { branch: "N/A", commit: "", percent: 100 } end # Adjust remote build data values for printing. r = { branch: (remote[:branch] + " " * 8)[0, 8], percent: remote[:percent].round(8), commit: remote[:commit][0, 8], } # Calculate coverage difference between the current and previous build. diff = (build_percent - remote[:percent]) pass = (diff > -THRESHOLD) puts puts "=" * 37 puts "COVERAGE RESULTS" puts "This build: #{build_percent.round(8)}% #{CURRENT_COMMIT[0, 8]}" puts "#{r[:branch]} build: #{r[:percent]}% #{r[:commit]}" puts "=" * 37 puts "Difference: #{diff.round(8)}%" puts "Pass? #{pass ? "yes" : "no"}" puts print_summary_text(build_percent, remote, pull_request_data) exit (pass || exit_0?) ? 0 : 1 end end