Stop using external parser services

.circleci/config.yml

@@ -5,7 +5,7 @@ orbs:
 executors:
   bibliothecary:
     docker:
-      - image: cimg/ruby:3.0.7
+      - image: cimg/ruby:3.2.4
     working_directory: ~/bibliothecary
 
 

.ruby-version

@@ -1 +1 @@
-3.0.7
+3.2.4

CHANGELOG.md

@@ -13,6 +13,18 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Removed
 
+## [12.0.0] - 2024-11-2?
+
+### Removed
+
+- This is a MAJOR release in that it removes support for hackage, carthage, hex, clojar, and swiftpm
+  from Bibliothecary. We are no longer doing any network calls when using Bibliothecary and reimplementing
+  parsing for those file types natively is non-trivial. Patches welcome :-)
+
+### Changed
+
+- Rewrote conda and yarn parsers to be in process vs calling out over the network
+
 ## [11.0.0] - 2024-11-22
 
 ### Changed

bibliothecary.gemspec

@@ -31,7 +31,6 @@ Gem::Specification.new do |spec|
   spec.add_development_dependency "rake", "~> 12.0"
   spec.add_development_dependency "rspec", "~> 3.0"
   spec.add_development_dependency "webmock"
-  spec.add_development_dependency "vcr"
   spec.add_development_dependency "rubocop"
   spec.add_development_dependency "rubocop-rails"
 end

lib/bibliothecary/configuration.rb

@@ -5,7 +5,6 @@ class Configuration
     attr_accessor :carthage_parser_host
     attr_accessor :clojars_parser_host
     attr_accessor :mix_parser_host
-    attr_accessor :yarn_parser_host
     attr_accessor :conda_parser_host
     attr_accessor :swift_parser_host
     attr_accessor :cabal_parser_host
@@ -16,8 +15,6 @@ def initialize
       @carthage_parser_host = "https://carthage.libraries.io"
       @clojars_parser_host  = "https://clojars.libraries.io"
       @mix_parser_host      = "https://mix.libraries.io"
-      @yarn_parser_host     = "https://yarn-parser.libraries.io"
-      @conda_parser_host    = "https://conda-parser.libraries.io"
       @swift_parser_host    = "http://swift.libraries.io"
       @cabal_parser_host    = "http://cabal.libraries.io"
     end

lib/bibliothecary/parsers/conda.rb

@@ -1,4 +1,4 @@
-require "json"
+require "yaml"
 
 module Bibliothecary
   module Parsers
@@ -15,14 +15,6 @@ def self.mapping
             parser: :parse_conda,
             kind: "manifest",
           },
-          match_filename("environment.yml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
-          match_filename("environment.yaml.lock") => {
-            parser: :parse_conda_lockfile,
-            kind: "lockfile",
-          },
         }
       end
 
@@ -31,34 +23,63 @@ def self.mapping
       add_multi_parser(Bibliothecary::MultiParsers::Spdx)
 
       def self.parse_conda(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "manifest")
-      end
+        manifest = YAML.load(file_contents)
+        deps = manifest.dig("dependencies")
+        deps.map do |dep|
+          next unless dep.is_a? String # only deal with strings to skip parsing pip stuff
 
-      def self.parse_conda_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        parse_conda_with_kind(file_contents, "lockfile")
+          parsed = parse_name_requirement_from_matchspec(dep)
+          Dependency.new(**parsed.merge(type: "runtime"))
+        end.compact
       end
 
-      def self.parse_conda_with_kind(info, kind)
-        dependencies = call_conda_parser_web(info, kind)[kind.to_sym]
-        dependencies.map { |dep_kv| Dependency.new(**dep_kv.merge(type: "runtime")) }
-      end
+      def self.parse_name_requirement_from_matchspec(ms)
+        # simplified version of the implementation in conda to handle what we care about
+        # https://github.com/conda/conda/blob/main/conda/models/match_spec.py#L598
+        # (channel(/subdir):(namespace):)name(version(build))[key1=value1,key2=value2]
+        return if ms.end_with?("@")
 
-      private_class_method def self.call_conda_parser_web(file_contents, kind)
-        host = Bibliothecary.configuration.conda_parser_host
-        response = Typhoeus.post(
-          "#{host}/parse",
-          headers: {
-            ContentType: "multipart/form-data",
-          },
-          body: {
-            file: file_contents,
-            # Unfortunately we do not get the filename in the mapping parsers, so hardcoding the file name depending on the kind
-            filename: kind == "manifest" ? "environment.yml" : "environment.yml.lock",
-          }
-        )
-        raise Bibliothecary::RemoteParsingError.new("Http Error #{response.response_code} when contacting: #{host}/parse", response.response_code) unless response.success?
+        # strip off comments and optional features
+        ms = ms.split(/#/, 2).first
+        ms = ms.split(/ if /, 2).first
+
+        # strip off brackets
+        ms = ms.match(/^(.*)(?:\[(.*)\])?$/)[1]
 
-        JSON.parse(response.body, symbolize_names: true)
+        # strip off any parens
+        ms = ms.match(/^(.*)(?:(\(.*\)))?$/)[1]
+
+        # deal with channel and namespace, I wish there was rsplit in ruby
+        split = ms.reverse.split(":", 2)
+        ms = split.last.reverse
+
+        # split the name from the version/build combo
+        matches = ms.match(/([^ =<>!~]+)?([><!=~ ].+)?/)
+        name = matches[1]
+        version_build = matches[2]
+
+        version = nil
+        if matches && matches[2]
+          version_build = matches[2]
+          # and now deal with getting the version from version/build
+          matches = version_build.match(/((?:.+?)[^><!,|]?)(?:(?<![=!|,<>~])(?:[ =])([^-=,|<>~]+?))?$/)
+          version = if matches
+            matches[1].strip
+          else
+            version_build.strip
+          end
+        end
+        # if it's an exact requirement, lose the =
+        if version&.start_with?("==")
+          version = version[2..]
+        elsif version&.start_with?("=")
+            version = version[1..]
+        end
+
+        return {
+          name: name,
+          requirement: version || "", # NOTE: this ignores build info
+        }
       end
     end
   end

lib/bibliothecary/parsers/go.rb

@@ -105,7 +105,7 @@ def self.parse_glide_yaml(file_contents, options: {}) # rubocop:disable Lint/Unu
       end
 
       def self.parse_glide_lockfile(file_contents, options: {}) # rubocop:disable Lint/UnusedMethodArgument
-        manifest = YAML.load file_contents
+        manifest = YAML.load(file_contents, permitted_classes: [Time])
         map_dependencies(manifest, "imports", "name", "version", "runtime")
       end