2018-11-05 18:19:29 -07:00
|
|
|
COVERAGE_FILE_PATH = "./coverage_fe/index.html"
|
2020-01-13 12:37:37 -07:00
|
|
|
JSON_COVERAGE_FILE_PATH = "./coverage_fe/coverage-final.json"
|
2019-07-12 09:16:47 -06:00
|
|
|
THRESHOLD = 0.001
|
|
|
|
REPO_URL = "https://api.github.com/repos/Farmbot/Farmbot-Web-App"
|
2019-02-27 15:17:47 -07:00
|
|
|
LATEST_COV_URL = "https://coveralls.io/github/FarmBot/Farmbot-Web-App.json"
|
2019-08-23 15:19:02 -06:00
|
|
|
COV_API_BUILDS_PER_PAGE = 5
|
2019-12-12 12:51:19 -07:00
|
|
|
COV_BUILDS_TO_FETCH = 20
|
2019-07-12 09:16:47 -06:00
|
|
|
PULL_REQUEST = ENV.fetch("CIRCLE_PULL_REQUEST", "/0")
|
2019-02-15 16:46:27 -07:00
|
|
|
CURRENT_BRANCH = ENV.fetch("CIRCLE_BRANCH", "staging") # "staging" or "pull/11"
|
2019-12-12 12:51:19 -07:00
|
|
|
BASE_BRANCHES = ["master", "staging"]
|
2018-11-05 18:19:29 -07:00
|
|
|
CURRENT_COMMIT = ENV.fetch("CIRCLE_SHA1", "")
|
2019-07-12 09:16:47 -06:00
|
|
|
CSS_SELECTOR = ".fraction"
|
2018-11-05 16:20:35 -07:00
|
|
|
FRACTION_DELIM = "/"
|
|
|
|
|
|
|
|
# Fetch JSON over HTTP. Rails probably already has a helper for this :shrug:
|
|
|
|
def open_json(url)
|
2019-02-13 17:28:15 -07:00
|
|
|
begin
|
2020-01-03 10:18:31 -07:00
|
|
|
JSON.parse(URI.open(url).read)
|
2019-10-30 12:24:29 -06:00
|
|
|
rescue *[OpenURI::HTTPError, SocketError] => exception
|
2019-02-13 17:28:15 -07:00
|
|
|
puts exception.message
|
|
|
|
return {}
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
2019-06-26 13:36:34 -06:00
|
|
|
# Don't fail on staging builds (i.e., not a pull request) to allow auto-deploys.
|
|
|
|
def exit_0?
|
|
|
|
CURRENT_BRANCH == "staging"
|
|
|
|
end
|
|
|
|
|
|
|
|
# Get pull request number. Return 0 if not a PR.
|
|
|
|
def pr_number
|
|
|
|
PULL_REQUEST.split("/")[-1].to_i
|
|
|
|
end
|
|
|
|
|
2019-02-14 21:09:39 -07:00
|
|
|
# Get pull request information from the GitHub API.
|
|
|
|
def fetch_pull_data()
|
2019-06-26 13:36:34 -06:00
|
|
|
if pr_number != 0
|
|
|
|
return open_json("#{REPO_URL}/pulls/#{pr_number}")
|
2019-02-13 18:21:41 -07:00
|
|
|
end
|
2019-02-14 21:09:39 -07:00
|
|
|
return {}
|
|
|
|
end
|
|
|
|
|
|
|
|
# Determine the base branch of the current build.
|
2019-12-12 12:51:19 -07:00
|
|
|
def get_base_branch(pull_data)
|
|
|
|
current_branch = BASE_BRANCHES.empty? ||
|
|
|
|
BASE_BRANCHES.include?(CURRENT_BRANCH) ? CURRENT_BRANCH : "staging"
|
2020-04-23 17:49:00 -06:00
|
|
|
provided_base_branch =
|
|
|
|
CURRENT_BRANCH.start_with?("master-hotfix/") ? "master" : nil;
|
|
|
|
pull_data.dig("base", "ref") || provided_base_branch || current_branch
|
2019-02-13 18:21:41 -07:00
|
|
|
end
|
|
|
|
|
2019-02-27 19:28:45 -07:00
|
|
|
# Gather relevant coverage data.
|
|
|
|
def relevant_data(build)
|
|
|
|
{ branch: build["branch"],
|
2019-07-12 09:16:47 -06:00
|
|
|
commit: build["commit_sha"],
|
|
|
|
percent: build["covered_percent"] }
|
2019-02-27 19:28:45 -07:00
|
|
|
end
|
|
|
|
|
2019-06-26 13:36:34 -06:00
|
|
|
# Fetch relevant coverage build data from commit.
|
|
|
|
def fetch_build_data_from_commit(commit)
|
|
|
|
if commit.nil?
|
|
|
|
puts "Commit not found."
|
|
|
|
build_data = {}
|
|
|
|
else
|
|
|
|
build_data = open_json("https://coveralls.io/builds/#{commit}.json")
|
|
|
|
end
|
|
|
|
return relevant_data(build_data)
|
|
|
|
end
|
|
|
|
|
2019-02-27 19:28:45 -07:00
|
|
|
# Fetch relevant remote coverage data for the latest commit on a branch.
|
|
|
|
def fetch_latest_branch_build(branch)
|
|
|
|
github_data = open_json("#{REPO_URL}/git/refs/heads/#{branch}")
|
|
|
|
if github_data.is_a? Array
|
|
|
|
github_data = {} # Didn't match a branch
|
|
|
|
end
|
|
|
|
commit = github_data.dig("object", "sha")
|
2019-06-26 13:36:34 -06:00
|
|
|
return fetch_build_data_from_commit(commit)
|
|
|
|
end
|
|
|
|
|
|
|
|
# Fetch latest remote coverage data for a branch (commit fetched via GH PR API).
|
|
|
|
def fetch_latest_pr_base_branch_build(branch)
|
|
|
|
github_data = open_json("#{REPO_URL}/pulls?state=closed&base=#{branch}")
|
|
|
|
commit = (github_data[0] || {}).dig("base", "sha")
|
|
|
|
return fetch_build_data_from_commit(commit)
|
2019-02-27 19:28:45 -07:00
|
|
|
end
|
|
|
|
|
2019-02-27 15:17:47 -07:00
|
|
|
# Fetch a page of build coverage report results.
|
|
|
|
def fetch_builds_for_page(page_number)
|
2019-10-30 12:24:29 -06:00
|
|
|
open_json("#{LATEST_COV_URL}?page=#{page_number}")["builds"] || []
|
2019-02-13 17:28:15 -07:00
|
|
|
end
|
|
|
|
|
2019-08-23 15:19:02 -06:00
|
|
|
# Number of coverage build data pages required to fetch the desired build count.
|
|
|
|
def cov_pages_required
|
|
|
|
(COV_BUILDS_TO_FETCH / COV_API_BUILDS_PER_PAGE.to_f).ceil
|
|
|
|
end
|
|
|
|
|
|
|
|
# Fetch coverage data from the last COV_BUILDS_TO_FETCH builds.
|
2019-02-27 15:17:47 -07:00
|
|
|
def fetch_build_data()
|
|
|
|
build_data = fetch_builds_for_page(1)
|
2019-08-23 15:19:02 -06:00
|
|
|
for page_number in 2..cov_pages_required
|
|
|
|
build_data.push(*fetch_builds_for_page(page_number))
|
|
|
|
end
|
2019-02-27 15:17:47 -07:00
|
|
|
clean_build_data = build_data
|
2019-07-12 09:16:47 -06:00
|
|
|
.reject { |build| build["covered_percent"].nil? }
|
|
|
|
.reject { |build| build["branch"].include? "/" }
|
2019-08-23 15:19:02 -06:00
|
|
|
puts "Using data from #{clean_build_data.length} of #{build_data.length}" \
|
|
|
|
" recent coverage builds."
|
2019-07-12 09:16:47 -06:00
|
|
|
clean_build_data.map { |build| relevant_data(build) }
|
2019-02-27 15:17:47 -07:00
|
|
|
end
|
|
|
|
|
|
|
|
# Print history and return the most recent match for the provided branch.
|
|
|
|
def latest_build_data(build_history, branch)
|
|
|
|
if branch == "*"
|
|
|
|
branch_builds = build_history
|
|
|
|
else
|
2019-07-12 09:16:47 -06:00
|
|
|
branch_builds = build_history.select { |build| build[:branch] == branch }
|
2019-02-27 15:17:47 -07:00
|
|
|
end
|
|
|
|
if branch_builds.length > 0
|
2019-06-26 13:36:34 -06:00
|
|
|
puts "\nCoverage history (newest to oldest):"
|
2019-07-12 09:16:47 -06:00
|
|
|
branch_builds.map { |build|
|
|
|
|
puts "#{build[:branch]}: #{build[:percent].round(3)}%"
|
|
|
|
}
|
|
|
|
branch_builds[0]
|
2019-02-27 15:17:47 -07:00
|
|
|
else
|
2019-07-12 09:16:47 -06:00
|
|
|
{ branch: branch, commit: nil, percent: nil }
|
2019-02-27 15:17:47 -07:00
|
|
|
end
|
2019-02-14 21:09:39 -07:00
|
|
|
end
|
|
|
|
|
2020-01-13 12:37:37 -07:00
|
|
|
# Calculate coverage results from JSON coverage report.
|
|
|
|
def get_json_coverage_results()
|
|
|
|
results = {lines: {covered: 0, total: 0}, branches: {covered: 0, total: 0}}
|
2020-01-24 10:10:46 -07:00
|
|
|
begin
|
|
|
|
data = open_json(JSON_COVERAGE_FILE_PATH)
|
|
|
|
rescue Errno::ENOENT
|
|
|
|
return results
|
|
|
|
end
|
2020-01-13 12:37:37 -07:00
|
|
|
data.each do |filename, file_coverage|
|
|
|
|
lineMap = {}
|
|
|
|
file_coverage["s"].each do |statement, count|
|
|
|
|
line = file_coverage["statementMap"][statement]["start"]["line"]
|
|
|
|
if lineMap[line].nil? || lineMap[line] < count
|
|
|
|
lineMap[line] = count
|
|
|
|
end
|
|
|
|
end
|
|
|
|
results[:lines][:covered] += lineMap.map{ |line, count| count }
|
|
|
|
.filter{ |count| count != 0}.length
|
|
|
|
results[:lines][:total] += lineMap.length
|
|
|
|
|
|
|
|
branches = []
|
|
|
|
file_coverage["b"].each do |branch, counts|
|
|
|
|
counts.map{ |count| branches.push(count) }
|
|
|
|
end
|
|
|
|
results[:branches][:covered] += branches.filter{ |count| count != 0 }.length
|
|
|
|
results[:branches][:total] += branches.length
|
|
|
|
end
|
|
|
|
results
|
|
|
|
end
|
|
|
|
|
2019-02-14 21:09:39 -07:00
|
|
|
# <commit hash> on <username>:<branch>
|
|
|
|
def branch_info_string?(target, pull_data)
|
|
|
|
unless pull_data.dig(target, "sha").nil?
|
|
|
|
"#{pull_data.dig(target, "sha")} on #{pull_data.dig(target, "label")}"
|
|
|
|
end
|
|
|
|
end
|
|
|
|
|
|
|
|
# Print a coverage difference summary string.
|
|
|
|
def print_summary_text(build_percent, remote, pull_data)
|
2019-03-01 13:29:27 -07:00
|
|
|
diff = (build_percent - remote[:percent]).round(3)
|
2019-02-14 21:09:39 -07:00
|
|
|
direction = diff > 0 ? "increased" : "decreased"
|
|
|
|
description = diff == 0 ? "remained the same at" : "#{direction} (#{diff}%) to"
|
2019-07-12 09:16:47 -06:00
|
|
|
puts "Coverage #{description} #{build_percent.round(3)}%" \
|
|
|
|
" when pulling #{branch_info_string?("head", pull_data)}" \
|
|
|
|
" into #{branch_info_string?("base", pull_data) || remote[:branch]}."
|
2018-11-05 16:20:35 -07:00
|
|
|
end
|
|
|
|
|
2018-11-06 11:11:58 -07:00
|
|
|
def to_percent(pair)
|
|
|
|
return ((pair.head / pair.tail) * 100).round(4)
|
|
|
|
end
|
|
|
|
|
2018-11-05 16:20:35 -07:00
|
|
|
namespace :coverage do
|
2019-07-12 09:16:47 -06:00
|
|
|
desc "Verify code test coverage changes remain within acceptable thresholds." \
|
|
|
|
"Compares current test coverage percentage from Jest output to previous" \
|
|
|
|
"values from the base branch of a PR (or the build branch if not a PR)." \
|
|
|
|
"This task is used during ci to fail PR builds if test coverage" \
|
|
|
|
"decreases significantly and can also be run locally after running" \
|
|
|
|
"`jest --coverage` or `npm test-slow`." \
|
|
|
|
"The Coveralls stats reporter used to perform this check, but didn't" \
|
2019-06-26 13:36:34 -06:00
|
|
|
"compare against a PR's base branch and would always return 0% change."
|
2018-11-05 16:20:35 -07:00
|
|
|
task run: :environment do
|
2020-01-24 10:10:46 -07:00
|
|
|
begin
|
|
|
|
# Fetch current build coverage data from the HTML summary.
|
|
|
|
statements, branches, functions, lines =
|
|
|
|
Nokogiri::HTML(URI.open(COVERAGE_FILE_PATH))
|
|
|
|
.css(CSS_SELECTOR)
|
|
|
|
.map(&:text)
|
|
|
|
.map { |x| x.split(FRACTION_DELIM).map(&:to_f) }
|
|
|
|
.map { |x| Pair.new(*x) }
|
|
|
|
rescue Errno::ENOENT
|
|
|
|
end
|
2018-11-05 16:20:35 -07:00
|
|
|
|
2020-01-13 12:37:37 -07:00
|
|
|
puts "\nUnable to determine coverage from HTML report." if lines.nil?
|
|
|
|
puts "Checking JSON report..." if lines.nil?
|
|
|
|
results = get_json_coverage_results()
|
|
|
|
lines_json_report = Pair.new(
|
|
|
|
results[:lines][:covered].to_f, results[:lines][:total].to_f)
|
|
|
|
branches_json_report = Pair.new(
|
|
|
|
results[:branches][:covered].to_f, results[:branches][:total].to_f)
|
|
|
|
if results[:lines][:total] > 0
|
|
|
|
lines = lines || lines_json_report
|
|
|
|
branches = branches || branches_json_report
|
|
|
|
end
|
|
|
|
covered = lines_json_report.head + branches_json_report.head
|
|
|
|
total = lines_json_report.tail + branches_json_report.tail
|
|
|
|
puts "JSON report aggregate: #{covered / total * 100}%"
|
|
|
|
puts
|
|
|
|
|
|
|
|
fallback_fraction = Pair.new(0.0, 1.0)
|
|
|
|
puts "\nUnable to determine coverage from build." if lines.nil?
|
2020-01-09 18:30:59 -07:00
|
|
|
statements = statements || fallback_fraction
|
|
|
|
branches = branches || fallback_fraction
|
|
|
|
functions = functions || fallback_fraction
|
|
|
|
lines = lines || fallback_fraction
|
|
|
|
|
2018-11-06 11:11:58 -07:00
|
|
|
puts
|
|
|
|
puts "This build: #{CURRENT_COMMIT}"
|
|
|
|
puts "Statements: #{to_percent(statements)}%"
|
|
|
|
puts "Branches: #{to_percent(branches)}%"
|
|
|
|
puts "Functions: #{to_percent(functions)}%"
|
|
|
|
puts "Lines: #{to_percent(lines)}%"
|
|
|
|
|
2019-02-13 17:28:15 -07:00
|
|
|
# Calculate an aggregate coverage percentage for the current build.
|
2019-07-12 09:16:47 -06:00
|
|
|
covered = lines.head + branches.head
|
|
|
|
total = lines.tail + branches.tail
|
2018-11-05 18:19:29 -07:00
|
|
|
build_percent = (covered / total) * 100
|
2019-02-13 17:28:15 -07:00
|
|
|
puts "Aggregate: #{build_percent.round(4)}%"
|
|
|
|
puts
|
2018-11-05 16:20:35 -07:00
|
|
|
|
2019-02-27 15:17:47 -07:00
|
|
|
# Fetch remote build coverage data for the current branch.
|
2019-02-14 21:09:39 -07:00
|
|
|
pull_request_data = fetch_pull_data()
|
2019-02-27 15:17:47 -07:00
|
|
|
coverage_history_data = fetch_build_data()
|
2019-02-13 17:28:15 -07:00
|
|
|
|
2019-02-27 15:17:47 -07:00
|
|
|
# Use fetched data.
|
2019-12-12 12:51:19 -07:00
|
|
|
base_branch = get_base_branch(pull_request_data)
|
|
|
|
remote = latest_build_data(coverage_history_data, base_branch)
|
2018-11-05 16:20:35 -07:00
|
|
|
|
2019-02-27 19:28:45 -07:00
|
|
|
if remote[:percent].nil?
|
2019-12-12 12:51:19 -07:00
|
|
|
puts "Coveralls data for '#{base_branch}' not found within history."
|
2019-06-26 13:36:34 -06:00
|
|
|
puts "Attempting to get coveralls build data for latest commit."
|
2019-12-12 12:51:19 -07:00
|
|
|
remote = fetch_latest_branch_build(base_branch)
|
2019-02-27 19:28:45 -07:00
|
|
|
end
|
|
|
|
|
2019-06-26 13:36:34 -06:00
|
|
|
if remote[:percent].nil?
|
2019-12-12 12:51:19 -07:00
|
|
|
puts "Coverage data for latest '#{base_branch}' commit not available."
|
2019-06-26 13:36:34 -06:00
|
|
|
puts "Attempting to use data from the previous commit (latest PR base)."
|
2019-12-12 12:51:19 -07:00
|
|
|
remote = fetch_latest_pr_base_branch_build(base_branch)
|
2019-06-26 13:36:34 -06:00
|
|
|
end
|
|
|
|
|
2019-12-12 12:51:19 -07:00
|
|
|
if remote[:percent].nil? && base_branch != "staging"
|
|
|
|
puts "Error getting coveralls data for '#{base_branch}'."
|
2019-08-23 15:19:02 -06:00
|
|
|
puts "Attempting to use staging build coveralls data from history."
|
2019-02-27 15:17:47 -07:00
|
|
|
remote = latest_build_data(coverage_history_data, "staging")
|
2018-11-06 11:11:58 -07:00
|
|
|
end
|
2018-11-05 16:20:35 -07:00
|
|
|
|
2019-02-13 17:28:15 -07:00
|
|
|
if remote[:percent].nil?
|
|
|
|
puts "Error getting coveralls data for staging."
|
2019-08-23 15:19:02 -06:00
|
|
|
puts "Attempting to use latest build coveralls data in history."
|
2019-02-27 15:17:47 -07:00
|
|
|
remote = latest_build_data(coverage_history_data, "*")
|
2019-02-13 17:28:15 -07:00
|
|
|
end
|
|
|
|
|
|
|
|
if remote[:percent].nil?
|
|
|
|
puts "Error getting coveralls data."
|
|
|
|
puts "Using 100 instead of nil for remote coverage value."
|
2019-07-12 09:16:47 -06:00
|
|
|
remote = { branch: "N/A", commit: "", percent: 100 }
|
2019-02-13 17:28:15 -07:00
|
|
|
end
|
|
|
|
|
|
|
|
# Adjust remote build data values for printing.
|
|
|
|
r = {
|
2019-07-12 09:16:47 -06:00
|
|
|
branch: (remote[:branch] + " " * 8)[0, 8],
|
2019-02-13 17:28:15 -07:00
|
|
|
percent: remote[:percent].round(8),
|
2019-07-12 09:16:47 -06:00
|
|
|
commit: remote[:commit][0, 8],
|
|
|
|
}
|
2019-02-13 17:28:15 -07:00
|
|
|
|
|
|
|
# Calculate coverage difference between the current and previous build.
|
|
|
|
diff = (build_percent - remote[:percent])
|
2018-11-06 11:52:28 -07:00
|
|
|
pass = (diff > -THRESHOLD)
|
|
|
|
|
2019-02-13 17:28:15 -07:00
|
|
|
puts
|
2018-11-05 16:20:35 -07:00
|
|
|
puts "=" * 37
|
|
|
|
puts "COVERAGE RESULTS"
|
2019-07-12 09:16:47 -06:00
|
|
|
puts "This build: #{build_percent.round(8)}% #{CURRENT_COMMIT[0, 8]}"
|
2019-02-13 17:28:15 -07:00
|
|
|
puts "#{r[:branch]} build: #{r[:percent]}% #{r[:commit]}"
|
2018-11-05 16:20:35 -07:00
|
|
|
puts "=" * 37
|
2019-02-13 17:28:15 -07:00
|
|
|
puts "Difference: #{diff.round(8)}%"
|
|
|
|
puts "Pass? #{pass ? "yes" : "no"}"
|
|
|
|
puts
|
2018-11-05 16:20:35 -07:00
|
|
|
|
2019-02-14 21:09:39 -07:00
|
|
|
print_summary_text(build_percent, remote, pull_request_data)
|
|
|
|
|
2019-06-26 13:36:34 -06:00
|
|
|
exit (pass || exit_0?) ? 0 : 1
|
2018-11-05 16:20:35 -07:00
|
|
|
end
|
|
|
|
end
|