[neon/infrastructure/pangea-gemstash] /: initial port from geminabox to gemstash
Carlos De Maine
null at kde.org
Fri May 26 00:23:02 BST 2023
Git commit e53d78535ca16f28fddcec614e0cdb37020e1f2e by Carlos De Maine.
Committed on 25/05/2023 at 23:22.
Pushed by carlosdem into branch 'master'.
initial port from geminabox to gemstash
A +11 -0 Gemfile
A +63 -0 Gemfile.backup
A +15 -0 Gemfile.lock
A +67 -0 Gemfile.lock.old
A +43 -0 Jenkinsfile
A +53 -0 README.md
A +110 -0 build_gem.rb
A +40 -0 config.rb
A +19 -0 config.yaml
A +15 -0 mangler.template
A +43 -0 start.sh
https://invent.kde.org/neon/infrastructure/pangea-gemstash/-/commit/e53d78535ca16f28fddcec614e0cdb37020e1f2e
diff --git a/Gemfile b/Gemfile
new file mode 100644
index 0000000..030630e
--- /dev/null
+++ b/Gemfile
@@ -0,0 +1,11 @@
+# ./Gemfile
+require "cgi"
+source "https://gem.cache.pangea.pub"
+source "http://localhost:9292"
+gem "releaseme"
+gem "jenkins_junit_builder"
+
+source "https://gem.cache.pangea.pub/upstream/#{CGI.escape("https://rubygems.org")}" do
+#gem "farraday"
+gem "gemstash"
+end
diff --git a/Gemfile.backup b/Gemfile.backup
new file mode 100644
index 0000000..40bbba8
--- /dev/null
+++ b/Gemfile.backup
@@ -0,0 +1,63 @@
+# ./Gemfile
+require "cgi"
+source "https://gem.cache.pangea.pub"
+gem "geminabox"
+gem "jenkins_junit_builder"
+
+source "https://gem.cache.pangea.pub/upstream/#{CGI.escape("https://my.gem-source.local")}" do
+gem "activesupport", "6.1.4.4"
+gem "concurrent-ruby", "~> 1.0" ">= 1.0.2"
+gem "i18n", ">= 1.6", "< 2"
+gem "minitest", ">= 5.1"
+gem "tzinfo", "~> 2.0"
+gem "zeitwerk", "~> 2.3"
+gem "concurrent-ruby", "1.2.2"
+gem "dalli", "3.2.4"
+gem "faraday", "0.17.6"
+gem "multipart-post", ">= 1.2", "< 3"
+gem "faraday_middleware", "0.14.0"
+gem "faraday", ">= 0.7.4", "< 1.0"
+gem "gemstash", "2.2.2"
+gem "activesupport", ">= 4.2", "< 8"
+gem "dalli", ">= 3.2.3, < 4"
+gem "faraday", "~> 0.9"
+gem "faraday_middleware", "~> 0.10"
+gem "lru_redux", "~> 1.1"
+gem "psych", ">= 3.2.1"
+gem "puma", "~> 4.0"
+gem "sequel", "~> 5.0"
+gem "server_health_check-rack", "~> 0.1"
+gem "sinatra", ">= 1.4, < 3.0"
+gem "sqlite3", "~> 1.3"
+gem "thor", "~> 0.20"
+gem "i18n", "1.10.0"
+gem "concurrent-ruby", "~> 1.0"
+gem "lru_redux", "1.1.0"
+gem "minitest", "5.18.0"
+gem "multipart-post", "2.1.1"
+gem "mustermann", "2.0.2"
+gem "ruby2_keywords", "~> 0.0.1"
+gem "nio4r", "2.5.9"
+gem "psych", "3.3.0"
+gem "puma", "4.3.12"
+gem "nio4r", "~> 2.0"
+gem "rack", "2.2.7"
+gem "rack-protection", "2.2.4"
+gem "rack"
+gem "ruby2_keywords", "0.0.5"
+gem "sequel", "5.68.0"
+gem "server_health_check", "1.0.2"
+gem "server_health_check-rack", "0.1.0"
+gem "server_health_check", "~> 1.0", ">= 1.0.1"
+gem "sinatra", "2.2.4"
+gem "mustermann", "~> 2.0"
+gem "rack", "~> 2.2"
+gem "rack-protection", "= 2.2.4"
+gem "tilt", "~> 2.0"
+gem "sqlite3", "1.6.3-x86_64-linux"
+gem "thor", "0.20.3"
+gem "tilt", "2.0.11"
+gem "tzinfo", "2.0.4"
+gem "concurrent-ruby", "~> 1.0"
+gem "zeitwerk", "2.5.4"
+end
diff --git a/Gemfile.lock b/Gemfile.lock
new file mode 100644
index 0000000..2498ad6
--- /dev/null
+++ b/Gemfile.lock
@@ -0,0 +1,15 @@
+GEM
+ remote: https://gem.cache.pangea.pub/
+ specs:
+
+GEM
+ remote: https://gem.cache.pangea.pub/upstream/https%3A%2F%2Fmy.gem-source.local/
+ specs:
+
+PLATFORMS
+ x86_64-linux
+
+DEPENDENCIES
+
+BUNDLED WITH
+ 2.4.13
diff --git a/Gemfile.lock.old b/Gemfile.lock.old
new file mode 100644
index 0000000..f3f6de7
--- /dev/null
+++ b/Gemfile.lock.old
@@ -0,0 +1,67 @@
+GEM
+ remote: https://rubygems.org/
+ specs:
+ activesupport (6.1.4.4)
+ concurrent-ruby (~> 1.0, >= 1.0.2)
+ i18n (>= 1.6, < 2)
+ minitest (>= 5.1)
+ tzinfo (~> 2.0)
+ zeitwerk (~> 2.3)
+ concurrent-ruby (1.2.2)
+ dalli (3.2.4)
+ faraday (0.17.6)
+ multipart-post (>= 1.2, < 3)
+ faraday_middleware (0.14.0)
+ faraday (>= 0.7.4, < 1.0)
+ gemstash (2.2.2)
+ activesupport (>= 4.2, < 8)
+ dalli (>= 3.2.3, < 4)
+ faraday (~> 0.9)
+ faraday_middleware (~> 0.10)
+ lru_redux (~> 1.1)
+ psych (>= 3.2.1)
+ puma (~> 4.0)
+ sequel (~> 5.0)
+ server_health_check-rack (~> 0.1)
+ sinatra (>= 1.4, < 3.0)
+ sqlite3 (~> 1.3)
+ thor (~> 0.20)
+ i18n (1.10.0)
+ concurrent-ruby (~> 1.0)
+ lru_redux (1.1.0)
+ minitest (5.18.0)
+ multipart-post (2.1.1)
+ mustermann (2.0.2)
+ ruby2_keywords (~> 0.0.1)
+ nio4r (2.5.9)
+ psych (3.3.0)
+ puma (4.3.12)
+ nio4r (~> 2.0)
+ rack (2.2.7)
+ rack-protection (2.2.4)
+ rack
+ ruby2_keywords (0.0.5)
+ sequel (5.68.0)
+ server_health_check (1.0.2)
+ server_health_check-rack (0.1.0)
+ server_health_check (~> 1.0, >= 1.0.1)
+ sinatra (2.2.4)
+ mustermann (~> 2.0)
+ rack (~> 2.2)
+ rack-protection (= 2.2.4)
+ tilt (~> 2.0)
+ sqlite3 (1.6.3-x86_64-linux)
+ thor (0.20.3)
+ tilt (2.0.11)
+ tzinfo (2.0.4)
+ concurrent-ruby (~> 1.0)
+ zeitwerk (2.5.4)
+
+PLATFORMS
+ x86_64-linux
+
+DEPENDENCIES
+ gemstash
+
+BUNDLED WITH
+ 2.4.13
diff --git a/Jenkinsfile b/Jenkinsfile
new file mode 100644
index 0000000..e9047cb
--- /dev/null
+++ b/Jenkinsfile
@@ -0,0 +1,43 @@
+env.GEM_HOME = '/var/lib/jenkins/.gem/ruby/2.2.0'
+env.PATH = '/var/lib/jenkins/.gem/ruby/2.2.0/bin:/usr/local/bin:/usr/bin:/bin:/usr/local/games:/usr/games:/snap/bin'
+env.GEM_PATH = '/var/lib/jenkins/.gem/ruby/2.2.0:/var/lib/jenkins/.gems/bundler'
+
+parallel(
+ "git[releaseme]": {
+ cleanNode('master') {
+ git_clone 'https://github.com/blue-systems/pangea-geminabox', 'geminabox'
+ git_clone 'https://anongit.kde.org/releaseme', 'releaseme'
+ sh 'ls -lah'
+ sh 'ls -lah releaseme'
+ sh "ruby geminabox/build_gem.rb `pwd`/releaseme"
+ }
+ },
+ "git[jenkins_junit_builder]": {
+ cleanNode('master') {
+ git_clone 'https://github.com/blue-systems/pangea-geminabox', 'geminabox'
+ git_clone 'https://github.com/hsitter/jenkins_junit_builder', 'jenkins_junit_builder'
+ sh 'ls -lah'
+ sh 'ls -lah jenkins_junit_builder'
+ sh "ruby geminabox/build_gem.rb `pwd`/jenkins_junit_builder"
+ }
+ }
+)
+
+def git_clone(url, dir, branch = 'master') {
+ checkout([$class: 'GitSCM', branches: [[name: "*/${branch}"]], doGenerateSubmoduleConfigurations: false, extensions: [[$class: 'RelativeTargetDirectory', relativeTargetDir: dir]], submoduleCfg: [], userRemoteConfigs: [[url: url]]])
+}
+
+
+def cleanNode(label = null, body) {
+ node(label) {
+ try {
+ wrap([$class: 'AnsiColorBuildWrapper', colorMapName: 'xterm']) {
+ wrap([$class: 'TimestamperBuildWrapper']) {
+ body()
+ }
+ }
+ } finally {
+ step([$class: 'WsCleanup', cleanWhenFailure: true])
+ }
+ }
+}
diff --git a/README.md b/README.md
new file mode 100644
index 0000000..f950efd
--- /dev/null
+++ b/README.md
@@ -0,0 +1,53 @@
+# Purpose
+
+Gemstash is both a cache for remote servers such as https://rubygems.org, and
+a private gem source..
+
+For our purposes we use as an on-site cache of gems and also as native host of
+from-git built gems. Because Bundler's git gemming isn't working particularly
+well for use we're building gems we need from git elsewhere and push them
+into the box so bundler can treat these gems just like any other gem.
+
+# Provision
+
+Gemstash is provisioned by our kitchen.
+
+# start.sh
+
+It's the entry point of the systemd service. It does some initial startup
+updating to make sure the stack is up to date (and hopefully secure). Once
+updated start.sh will gem install and start gemstash.
+
+# config.yaml
+
+Gemstash looks for for config.yaml for configuration instructions. Puma is
+bundled as the (virtually unconfigurable) default web server for gemstash
+(which in turn is proxied by apache in deployment scenarios). config.yaml
+configures it and other db related options and is copied to
+~/.gemstash/config.yaml during the start.sh bootstrap. Also, an api key
+based authentication mechanism is added to the core functionality. The
+authentication credentials are stored in ~/.config/pangea_build_gem.yaml
+
+# Auth
+
+For API usage we use a key for use in ~/.gem/credentials on the client-side.
+
+# Git-builds
+
+Jenkinsfile and build_gem.rb facilitiate building of a gem from git. Jenkinsfile
+assembles the repos and uses build_gem to actually build and push the gem.
+
+## build_gem
+
+Building is fairly straight forward. We patch the `*.gemspec` found in the gem
+repo to hijack the version definition. We'll then append a `.$date.$time` suffix
+to differentiate our builds from potential upstream ones. Additionally this
+ensures different versions across rebuilds. The datetime itself is derived from
+the latest git commit and as such is persistent across rebuilds of the same
+HEAD (and hopefully monotone increasing).
+
+Once we have a gem we'll conduct some additional assertations to make sure our
+build is in fact newer than what is in the box already.
+
+If all is fine we `gem push` to our box. This requires ~/.gem/credentials to be
+set up with our api key.
diff --git a/build_gem.rb b/build_gem.rb
new file mode 100644
index 0000000..9a15470
--- /dev/null
+++ b/build_gem.rb
@@ -0,0 +1,110 @@
+# frozen_string_literal: true
+#
+# Copyright © 2017 Harald Sitter <sitter at kde.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License or (at your option) version 3 or any later version
+# accepted by the membership of KDE e.V. (or its successor approved
+# by the membership of KDE e.V.), which shall act as a proxy
+# defined in Section 14 of version 3 of the license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'fileutils'
+require 'rubygems/package'
+require 'tmpdir'
+require 'yaml'
+
+# Mangles a gemspec file so we can build it into something sane.
+class GemSpecMangler
+ attr_reader :path
+ attr_reader :new_path
+
+ def self.find_file(dir)
+ gemspecs = Dir.glob("#{dir}/*.gemspec")
+ raise "too many gemspecs #{gemspecs}" if gemspecs.size > 1
+ raise "couldnt find gemspec file in #{Dir.pwd}" if gemspecs.empty?
+ gemspecs[0]
+ end
+
+ def initialize(path = self.class.find_file)
+ @path = File.absolute_path(path)
+ @new_path = "#{path}.new"
+ end
+
+ def mangle!
+ injected = false
+ File.open(new_path, 'w') do |out|
+ File.open(path).each_line do |line|
+ if line.strip.start_with?('#') || injected
+ out.write(line)
+ next
+ end
+ # The line is the first line that is not a comment whilest not having
+ # injected our magic.
+ injected = true
+ # We mangle the spec data generated later in the gemspec file by
+ # disabling and all push restrictions and setting an ever increasing
+ # version
+ out.write(File.read("#{__dir__}/mangler.template"))
+ out.write(line)
+ end
+ end
+ end
+end
+
+dir = File.expand_path(ARGV[0])
+
+FileUtils.rm_rf('pangeapkg')
+spec = GemSpecMangler.new(GemSpecMangler.find_file(dir))
+spec.mangle!
+# This would be much nicer if we simply called gem build in the pwd where
+# we want the gem to be. BUT, shitty gemspecs that do not resolve paths
+# to their absolute variant will then fail to find assets.
+system('gem', 'build', spec.new_path, '-V', chdir: dir) || raise
+
+# The version of our gem is retained across builds and only changes to new
+# commits. To prevent us from uploading an already existing version we'll
+# attempt to fetch our gem. If we already uploaded the thing we'll have 1
+# gem and be able to exit.
+gem_files = Dir.glob("#{dir}/*.gem")
+raise "Too many gems! #{gemfiles}" unless gem_files.size == 1
+gem_file = gem_files.fetch(0)
+gem_spec = Gem::Package.new(gem_file).spec
+
+# Validate our spec against the one in the box (if it has one.)
+Dir.mktmpdir do |tmpdir|
+ system('gem', 'fetch',
+ '--prerelease',
+ '--clear-sources',
+ '--source', 'https://gem.cache.pangea.pub',
+ gem_spec.name,
+ chdir: tmpdir)
+ gems = Dir.glob("#{tmpdir}/*.gem")
+ # Should't have more than 1 obviously.
+ raise "Fetched too many gems #{gems}" if gems.size > 1
+
+ if !gems.empty? && File.basename(gems[0]) == File.basename(gem_file)
+ puts "Gem already exists in the box #{gem_files}"
+ exit
+ elsif !gems.empty?
+ # Our _new_ version should be greater than the fetched one.
+ other_spec = Gem::Package.new(gems.fetch(0)).spec
+ if gem_spec.version < other_spec.version
+ raise "Our new version is older than the one in the box!!!\n" \
+ "#{gem_spec} vs. #{other_spec}"
+ end
+ end
+end
+
+# .gem/credentials controls API_KEY used here.
+system('gem', 'push', gem_file,
+ '--host', 'https://gem.cache.pangea.pub') || raise
diff --git a/config.rb b/config.rb
new file mode 100644
index 0000000..f494ca9
--- /dev/null
+++ b/config.rb
@@ -0,0 +1,40 @@
+# frozen_string_literal: true
+#
+# Copyright © 2017 Harald Sitter <sitter at kde.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License or (at your option) version 3 or any later version
+# accepted by the membership of KDE e.V. (or its successor approved
+# by the membership of KDE e.V.), which shall act as a proxy
+# defined in Section 14 of version 3 of the license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+require 'yaml'
+
+# Configuration
+module Config
+ module_function
+
+ XDG_CONFIG_HOME = ENV.fetch('XDG_CONFIG_HOME', "#{Dir.home}/.config").freeze
+
+ def data
+ @data ||= YAML.load_file("#{XDG_CONFIG_HOME}/pangea_build_gem.yaml")
+ end
+
+ def method_missing(meth, *)
+ data.fetch(meth.to_s) || super
+ end
+
+ def respond_to_missing?(meth, *)
+ data.key?(meth.to_s) || super
+ end
+end
diff --git a/config.yaml b/config.yaml
new file mode 100644
index 0000000..fa65572
--- /dev/null
+++ b/config.yaml
@@ -0,0 +1,19 @@
+# ~/.gemstash/config.yml
+---
+:base_path: "~/.gemstash/"
+:cache_type: memory
+:cache_max_size: 2000
+#revisit when we are up and running if necessary
+#:cache_type: memcached
+#:memcached_servers: localhost:11211
+:db_adapter: sqlite3
+:db_url:
+:db_options:
+ :max_connections: 1
+#:rubygems_url: https://gem.cache.pangea.pub
+:ignore_gemfile_source: false
+:puma_threads: 32
+:bind: tcp://0.0.0.0:9292
+:protected_fetch: true
+:fetch_timeout: 10
+:log_file: gemstash.log
diff --git a/mangler.template b/mangler.template
new file mode 100644
index 0000000..869d304
--- /dev/null
+++ b/mangler.template
@@ -0,0 +1,15 @@
+require 'date'
+module SpecMangler
+ def initialize(*)
+ super do |spec|
+ yield spec
+ spec.metadata.delete('allowed_push_host')
+ commit_time = DateTime.parse(`git log -1 --date=iso --format=%cd`.strip).to_time.utc.strftime('%Y%m%d.%H%M')
+ raise unless $?.success? && commit_time
+ spec.version = spec.version.to_s + "." + commit_time
+ end
+ end
+end
+class Gem::Specification
+ prepend SpecMangler
+end
diff --git a/start.sh b/start.sh
new file mode 100755
index 0000000..339e304
--- /dev/null
+++ b/start.sh
@@ -0,0 +1,43 @@
+#!/bin/sh
+#
+# Copyright © 2017 Harald Sitter <sitter at kde.org>
+#
+# This program is free software; you can redistribute it and/or
+# modify it under the terms of the GNU General Public License as
+# published by the Free Software Foundation; either version 2 of
+# the License or (at your option) version 3 or any later version
+# accepted by the membership of KDE e.V. (or its successor approved
+# by the membership of KDE e.V.), which shall act as a proxy
+# defined in Section 14 of version 3 of the license.
+#
+# This program is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with this program. If not, see <http://www.gnu.org/licenses/>.
+
+# set up stack for making our own gems
+set -ex
+export GEM_HOME=$(ruby -e 'puts Gem.user_dir')
+export GEM_PATH=$GEM_HOME:$HOME/.gems/bundler
+export PATH=$GEM_HOME/bin:$PATH
+if [ ! -f ~/.gemrc ]; then
+ echo 'gem: --no-document' > ~/.gemrc
+ fi
+fi
+
+# install required system deps for gemstash
+apt install libsqlite3-dev
+#apt install memcached
+
+# install, verify and upgrade bundler
+gem install bundler --conservative
+gem update bundler
+bundle update --bundler
+
+# install, configure and start gemstash
+gem install gemstash
+cp -vf config.yaml .gemstash/config.yaml
+gem exec gemstash start --no-daemonize
More information about the Neon-commits
mailing list