summaryrefslogtreecommitdiffstats
path: root/fedora2spdx.rb
diff options
context:
space:
mode:
Diffstat (limited to 'fedora2spdx.rb')
-rw-r--r--fedora2spdx.rb133
1 files changed, 83 insertions, 50 deletions
diff --git a/fedora2spdx.rb b/fedora2spdx.rb
index 7cc2398..67bcbc2 100644
--- a/fedora2spdx.rb
+++ b/fedora2spdx.rb
@@ -4,11 +4,11 @@ require 'open3'
require 'rubygems'
require 'rubygems/package'
require 'tmpdir'
-require 'licensee'
-# Licences.csv are from validate_ruby_files.rb and it contains what was possible to gather from upstream gemfiles and Fedora specfiles.
-# This file exists to validate, that the output from gem2rpm for Fedora is the same. And possibly also validate Licensefiles, cos
-# the goddamn MIT and BSD have around 20 possibilities from callaway to spdx...
-csv = CSV.parse(File.read('licences.csv'), col_sep: ';', headers: true)
+# require 'licensee'
+
+# Constants
+
+CSV_HEADER = "gem2rpm_and_fedora_matches?;license_validate_exit_code;gem_name;fedora_license;gem2rpm_license;dead_package".freeze
class ThreadWorker
MAX_THREADS = Etc.nprocessors
@@ -57,21 +57,37 @@ class ThreadWorker
end
class << self
- def execute(command, pwd: nil)
+ def execute(command, pwd: nil, ret_exit: false)
options = {}
options[:chdir] = pwd if pwd
$stderr.puts "Executing: #{command}"
+ sleep 0.5
stdout, stderr, status = Open3.capture3(command, options)
- raise CommandError.new("Command failed: #{command}", stdout, stderr, status.exitstatus) unless status.success?
+ raise CommandError.new("Command failed: #{command}; #{stderr}; #{stdout}", stdout, stderr, status.exitstatus) unless status.success?
- status.exitstatus
+ if ret_exit
+ status.exitstatus
+ else
+ stdout
+ end
end
end
end
+csv = ThreadWorker.new(Dir['rubygem-*.spec']) do |slice|
+ slice.map do |file|
+ name, spec_license = nil, nil
+ file = File.read(file)
+ name = file.scan(/%(?:global|define)[[:space:]]+gem_name[[:space:]]+([A-Za-z0-9\-_.]+)/).flatten.first.strip
+ raise "name empty #{name} #{spec_license}" if name.empty?
+ spec_license = file.scan(/License:[[:space:]]*(.*)/).flatten.first.strip
+ [name, spec_license]
+ end
+end.gather_pool
+
# Fetch the Fedora sources
-ThreadWorker.new(csv.to_a[1..]) do |slice|
+ThreadWorker.new(csv) do |slice|
slice.map do |row|
name = "rubygem-#{row[0]}"
@@ -84,13 +100,22 @@ end.gather_pool
# Sigh... they have tar as rubygem source...
EXCLUDED_SOURCES = %w[rubygem-morph-cli rubygem-krb5-auth rubygem-asciidoctor rubygem-rgen rubygem-net-irc].freeze
+orphaned = []
+
# Fetch the gem from lookaside cache
-ThreadWorker.new(csv.to_a[1..]) do |slice|
+ThreadWorker.new(csv) do |slice|
slice.map do |row|
name = "rubygem-#{row[0]}"
dir = Dir["#{name}/*.gem"]
+ dead = Dir["#{name}/dead.package"]
+
+ unless dead.empty?
+ orphaned << name
+ next
+ end
+
next unless dir.empty?
next if EXCLUDED_SOURCES.include? name
@@ -100,7 +125,7 @@ ThreadWorker.new(csv.to_a[1..]) do |slice|
puts "sources for #{name}"
ThreadWorker.execute("fedpkg sources", pwd: name)
end
-end.gather_pool
+end.gather_pool.reject(&:nil?)
res = if File.exist?("gem2rpm.cache")
File.read("gem2rpm.cache").split("\n")
@@ -214,8 +239,18 @@ end
# end
# str + ";" + fedora_name + ";" + fedora_license.to_s + ";" + gem2rpm_license.to_s
# end
+# Somehow the following block changes contents of res, let's deep copy
res2 = Marshal.load Marshal.dump(res)
+license_check = ->(fedora_license) {
+ begin
+ status = ThreadWorker.execute("license-validate \"#{fedora_license}\"", ret_exit: true).to_s
+ "#{status}"
+ rescue ThreadWorker::CommandError => e
+ if e.status == 1 then "#{e.status}" else "#{e.stderr}######{e.stdout}" end
+ end
+}
+
ret = ThreadWorker.new(res)do |slice|
slice.map do |arr|
gem2rpm_name = arr[0]
@@ -223,41 +258,40 @@ ret = ThreadWorker.new(res)do |slice|
gem2rpm_license = arr[2]&.strip
gem2rpm_license_file = arr[3]
- fedora_gem = csv.find { |row| row["gem_name"] == gem2rpm_name }
- fedora_name = fedora_gem["gem_name"]
- fedora_license = fedora_gem["fedora_license"]&.strip
- fedora_gem_license = fedora_gem["gem_license"]
+ fedora_license = csv.find { |row| row[0] == gem2rpm_name }[1].strip
+ fedora_name = gem2rpm_name
+ # begin
+ # fedora_name = fedora_gem["gem_name"]
+ # rescue => e
+ # require 'irb';binding.irb
+ # end
+ # fedora_license = fedora_gem["fedora_license"]&.strip
+ # fedora_gem_license = fedora_gem["gem_license"]
- raise "The names of gems differ. Leading me to this is gem2rpm: #{gem2rpm_name} fedora: #{fedora_name}" if gem2rpm_name != fedora_name
+ # raise "The names of gems differ. Leading me to this is gem2rpm: #{gem2rpm_name} fedora: #{fedora_name}" if gem2rpm_name != fedora_name
str = ''
- if fedora_license == fedora_gem_license && fedora_license == gem2rpm_license
+ if fedora_license == gem2rpm_license
# Matches
str = true.to_s
- res = begin
- status = ThreadWorker.execute("license-validate \"#{fedora_license}\"").to_s
- "#{status}"
- rescue ThreadWorker::CommandError => e
- "#{e.status}"
- end
+ res = license_check.call(fedora_license)
str += ";" + res
str
else
# Doesn't match
- str = false.to_s + ";" + begin
- status = ThreadWorker.execute("license-validate \"#{fedora_license}\"").to_s
- "#{status}"
- rescue ThreadWorker::CommandError => e
- if e.status == 1 then "#{e.status}" else "#{e.stderr}######{e.stdout}" end
- end
+ str = false.to_s + ";" + license_check.call(fedora_license)
end
str + ";" + fedora_name + ";" + fedora_license.to_s + ";" + gem2rpm_license.to_s
end
end.gather_pool(2)
-final = ret.sort { |a, b| c = a.split(";"); d = b.split(";"); c[1] <=> d[1] }.unshift("gem2rpm_and_fedora_matches?;license_validate_exit_code;gem_name;fedora_license;gem2rpm_license")
+
+orphaned = orphaned.map { |package| "#{false};255;#{package};N/A;N/A;#{true}" }
+sort_by_license_check = ->(a,b) { c = a.split(";"); d = b.split(";"); c[1] <=> d[1] }
+
+final = ret.sort(&sort_by_license_check).union(orphaned).unshift(CSV_HEADER)
puts final
# The state of Fedora Rubygems (excl a few that dont have gem as their source in Fedora lookaside cache)
@@ -276,41 +310,40 @@ try_convert = ThreadWorker.new(res2) do |slice|
gem2rpm_license = arr[2]&.strip&.gsub(" and ", " AND ")&.gsub(" or ", " OR ")
gem2rpm_license_file = arr[3]
- fedora_gem = csv.find { |row| row["gem_name"] == gem2rpm_name }
- fedora_name = fedora_gem["gem_name"]
- fedora_license = fedora_gem["fedora_license"]&.strip&.gsub(" and ", " AND ")&.gsub(" or ", " OR ")
- fedora_gem_license = fedora_gem["gem_license"]
+ fedora_license = csv.find { |row| row[0] == gem2rpm_name }[1].strip
+ fedora_name = gem2rpm_name
+ # begin
+ # fedora_name = fedora_gem["gem_name"]
+ # rescue => e
+ # require 'irb';binding.irb
+ # end
+ # fedora_license = fedora_gem["fedora_license"]&.strip
+ # fedora_gem_license = fedora_gem["gem_license"]
- raise "The names of gems differ. Leading me to this is gem2rpm: #{gem2rpm_name} fedora: #{fedora_name}" if gem2rpm_name != fedora_name
+ # raise "The names of gems differ. Leading me to this is gem2rpm: #{gem2rpm_name} fedora: #{fedora_name}" if gem2rpm_name != fedora_name
+ fedora_license = fedora_license&.strip&.gsub(" and ", " AND ")&.gsub(" or ", " OR ")
+ # fedora_gem_license = fedora_gem["gem_license"]
+
+ # raise "The names of gems differ. Leading me to this is gem2rpm: #{gem2rpm_name} fedora: #{fedora_name}" if gem2rpm_name != fedora_name
str = ''
- if fedora_license == fedora_gem_license && fedora_license == gem2rpm_license
+ if fedora_license == gem2rpm_license
# Matches
str = true.to_s
- res = begin
- status = ThreadWorker.execute("license-validate \"#{fedora_license}\"").to_s
- "#{status}"
- rescue ThreadWorker::CommandError => e
- "#{e.status}"
- end
+ res = license_check.call(fedora_license)
str += ";" + res
str
else
# Doesn't match
- str = false.to_s + ";" + begin
- status = ThreadWorker.execute("license-validate \"#{fedora_license}\"").to_s
- "#{status}"
- rescue ThreadWorker::CommandError => e
- if e.status == 1 then "#{e.status}" else "#{e.stderr}######{e.stdout}" end
- end
+ str = false.to_s + ";" + license_check.call(fedora_license)
end
str + ";" + fedora_name + ";" + fedora_license.to_s + ";" + gem2rpm_license.to_s
end
end.gather_pool(2)
- .sort { |a, b| c = a.split(";"); d = b.split(";"); c[1] <=> d[1] }
- .unshift("gem2rpm_and_fedora_matches?;license_validate_exit_code;gem_name;fedora_license;gem2rpm_license")
+ .sort(&sort_by_license_check)
+ .unshift(CSV_HEADER)
File.write("rubygems_try_convert_conjunctions.csv", try_convert.join("\n"))
generated by cgit (git 2.34.1) at 2024-06-08 21:50:53 +0000