From 368ebfaba224923fcb605c1cb6c4a5af5b1a4c1f Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Sun, 3 Aug 2025 20:53:36 +0200 Subject: [PATCH 01/52] save --- bin/apas | 28 ++++++++ bin/cht | 98 +++++++++++++++++++++++++++ bin/index | 16 +++++ bin/zshrc/brew | 2 + bin/zshrc/init | 3 + home/.local/bin/{index => index-list} | 0 6 files changed, 147 insertions(+) create mode 100755 bin/apas create mode 100755 bin/cht create mode 100755 bin/index rename home/.local/bin/{index => index-list} (100%) diff --git a/bin/apas b/bin/apas new file mode 100755 index 0000000..6e2ebdb --- /dev/null +++ b/bin/apas @@ -0,0 +1,28 @@ +#!/usr/bin/env bash + +function lhash { + echo -n "${1}" | openssl dgst -sha512 | cut -d' ' -f2 | openssl dgst -md5 | cut -d' ' -f2 +} + +# Prompt user for input +read -p "Website: " url +read -p "Login: " login +read -s -p "Password: " password +echo + +domain=$(echo "$url" | sed -E 's~https?://([^/]+).*~\1~') +hash=$(lhash "$login") +pass_path="web/${domain}/${hash}" + +# Entry content +entry="${password} +url: ${url} +login: ${login} +" + +echo "$entry" | pass insert -m "$pass_path" + +# url +# login +# tags +# note \ No newline at end of file diff --git a/bin/cht b/bin/cht new file mode 100755 index 0000000..67ebfd0 --- /dev/null +++ b/bin/cht @@ -0,0 +1,98 @@ +#!/usr/bin/env node + +import path from "path" +import fs from "fs" + +// DAT_ROOT should exists in ~/.zshrc-local +const indexPath = path.join(process.env["NOTES_DIR"], "") +const processFile = path.join(process.cwd(), process.argv[2]) + + +const data = fs.readFileSync(processFile, 'utf8') +const jsonData = JSON.parse(data) + +function wantSave(str) { + return typeof str == "string" && str.split("\n").length > 10 +} + +function saveToFileSync(content, file) { + const dir = path.dirname(file); + try { + if (!fs.existsSync(dir)) { + fs.mkdirSync(dir, { recursive: true }) + } + + fs.writeFileSync(file, content, 'utf8') + } catch (err) { + console.error('Error saving file:', err) + } +} + +const toSave = jsonData.flatMap(o => { + const date = new Date(o.create_time * 1000) + + // const formattedDate = date.toLocaleDateString('en-GB', { + // day: '2-digit', + // month: '2-digit', + // year: '2-digit', + // }) + // return formattedDate + + const day = String(date.getDate()).padStart(2, '0') + const month = String(date.getMonth() + 1).padStart(2, '0') + const year = String(date.getFullYear()).slice(-2) + + let baseNotePath = path.join(indexPath, "gpt", `${day}-${month}-${year}`, `${o.title}`) + + let conversations = [] + for (const [key, value] of Object.entries(o.mapping)) { + //console.log(key, value) + + conversations.push({ + path: path.join(baseNotePath, `${key}.json`), + content: JSON.stringify(value, null, 2) + }) + + let parts = value?.message?.content.parts + if (Array.isArray(parts)) { + + if (parts.length == 0) { + // skip + } else if (parts.length == 1) { + let fileName = path.join(baseNotePath, `${key}.md`) + let md = parts[0] + + if (wantSave(md)) { + conversations.push({ + path: fileName, + content: md + }) + } + } else { + + for (const [key, value] of parts.entries()) { + let fileName = path.join(baseNotePath, `${key}-${key+1}.md`) + let md = parts[key] + + if (wantSave(md)) { + conversations.push({ + path: fileName, + content: md + }) + } + } + } + } + + //process.exit(-1) + + } + + return conversations +}) + + +for(const f of toSave) { +//for(const f of toSave.slice(0,5)) { + saveToFileSync(f.content,f.path) +} diff --git a/bin/index b/bin/index new file mode 100755 index 0000000..fd6a40d --- /dev/null +++ b/bin/index @@ -0,0 +1,16 @@ +#!/bin/bash + +# find $NOTES_DIR/gpt -type d -path "$NOTES_DIR/gpt/[0-9][0-9]-[0-9][0-9]-[0-9][0-9]/*" | while read -r path; do +# echo $(basename "$path") +# done | sort -u | fzf + +SEARCH_DIR="$NOTES_DIR/gpt" +DELIMITER="|" + +selected=$(find "$SEARCH_DIR" -type d -path "$SEARCH_DIR/[0-9][0-9]-[0-9][0-9]-[0-9][0-9]/*" | while read -r path; do + topic=$(basename "$path") + echo -e "$topic\t$path" +done | fzf --delimiter='\t' --with-nth=1) + +selected_path=$(echo "$selected" | cut -f2) +echo "Path: $selected_path" diff --git a/bin/zshrc/brew b/bin/zshrc/brew index 4e13a77..6e3f24d 100644 --- a/bin/zshrc/brew +++ b/bin/zshrc/brew @@ -6,4 +6,6 @@ if [[ "$OSTYPE" == "darwin"* ]]; then elif [[ "$ARCH" == "x86_64" ]]; then eval "$(/usr/local/bin/brew shellenv)" fi + + export PATH="$(brew --prefix ruby)/bin:$PATH" fi \ No newline at end of file diff --git a/bin/zshrc/init b/bin/zshrc/init index 8f7e2a3..c32fcf1 100644 --- a/bin/zshrc/init +++ b/bin/zshrc/init @@ -3,6 +3,9 @@ export PATH="$DAT_ROOT/bin:$HOME/.local/bin:$PATH" +export GEM_HOME="$HOME/.gems" +export GEM_PATH=$HOME/.gems + export RUBYLIB="$DAT_ROOT/lib" export PASSWORD_STORE_DIR=$HOME/.local/lib/secure-vault/passwords diff --git a/home/.local/bin/index b/home/.local/bin/index-list similarity index 100% rename from home/.local/bin/index rename to home/.local/bin/index-list From 0ef37ad846245252eec478f6a9a20eb67473baab Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 4 Aug 2025 09:44:22 +0200 Subject: [PATCH 02/52] Add command that retreive data from backend and use zstd compression to transfer --- bin/get | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100755 bin/get diff --git a/bin/get b/bin/get new file mode 100755 index 0000000..ebd45b1 --- /dev/null +++ b/bin/get @@ -0,0 +1,35 @@ +ARCHIVE=false +DESTINATION_LOCAL_DIR="./" + +if [[ "$1" == "--archive" ]]; then + ARCHIVE=true + shift +fi + +if [[ $# -lt 2 ]]; then + echo "Usage: $0 [--archive] user@host remote_source_dir [local_destination_dir]" + exit 1 +fi + +USER_HOST="$1" +REMOTE_USER="${USER_HOST%@*}" +HOST="${USER_HOST#*@}" + +REMOTE_SOURCE_DIR="$2" +if [[ -n "$3" ]]; then + DESTINATION_LOCAL_DIR="$3" +fi + +echo "ARCHIVE=$ARCHIVE" +echo "REMOTE_USER=$REMOTE_USER" +echo "HOST=$HOST" +echo "REMOTE_SOURCE_DIR=$REMOTE_SOURCE_DIR" +echo "DESTINATION_LOCAL_DIR=$DESTINATION_LOCAL_DIR" + + +# Example that works +# ssh debian@gurgul.org "tar --zstd -cf - -C /home/debian .dat" | tar --zstd -xf - -C . +# get debian@gurgul.org .dat + +TAR_COMMAND="tar --zstd -cf - -C /home/$REMOTE_USER $REMOTE_SOURCE_DIR" +ssh $REMOTE_USER@$HOST $TAR_COMMAND | tar --zstd -xf - -C $DESTINATION_LOCAL_DIR \ No newline at end of file From a4f3987b97efe4f8e418564db2301d4123c22fab Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 4 Aug 2025 10:46:03 +0200 Subject: [PATCH 03/52] Add possibility to download absolute dirs --- bin/get | 26 ++++++++++++++++++++++++-- 1 file changed, 24 insertions(+), 2 deletions(-) diff --git a/bin/get b/bin/get index ebd45b1..299ffb7 100755 --- a/bin/get +++ b/bin/get @@ -20,16 +20,38 @@ if [[ -n "$3" ]]; then DESTINATION_LOCAL_DIR="$3" fi +if [[ "$REMOTE_SOURCE_DIR" = /* ]]; then + IS_ABSOLUTE=true +else + IS_ABSOLUTE=false +fi + + echo "ARCHIVE=$ARCHIVE" echo "REMOTE_USER=$REMOTE_USER" echo "HOST=$HOST" echo "REMOTE_SOURCE_DIR=$REMOTE_SOURCE_DIR" echo "DESTINATION_LOCAL_DIR=$DESTINATION_LOCAL_DIR" +REMOTE_BASE_DIR="" + +if $IS_ABSOLUTE; then + REMOTE_BASE_DIR="/" +else + REMOTE_BASE_DIR=/home/$REMOTE_USER +fi + # Example that works # ssh debian@gurgul.org "tar --zstd -cf - -C /home/debian .dat" | tar --zstd -xf - -C . # get debian@gurgul.org .dat -TAR_COMMAND="tar --zstd -cf - -C /home/$REMOTE_USER $REMOTE_SOURCE_DIR" -ssh $REMOTE_USER@$HOST $TAR_COMMAND | tar --zstd -xf - -C $DESTINATION_LOCAL_DIR \ No newline at end of file +# get --archive debian@gurgul.org /etc + +TAR_COMMAND="tar --zstd -cf - -C $REMOTE_BASE_DIR $REMOTE_SOURCE_DIR" + +if $ARCHIVE; then + ssh $REMOTE_USER@$HOST $TAR_COMMAND > "$DESTINATION_LOCAL_DIR/$(basename "$REMOTE_SOURCE_DIR")-$(date +"%d-%m-%Y").tar.zst" +else + ssh $REMOTE_USER@$HOST $TAR_COMMAND | tar --zstd -xf - -C $DESTINATION_LOCAL_DIR +fi \ No newline at end of file From 68e1bac4b394352a2b9fee33a878128599de5cc2 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 4 Aug 2025 19:41:41 +0200 Subject: [PATCH 04/52] parse archive param --- lib/make.rb | 34 +++++++++++++++++++++++++++++++--- recipes/nginx/generate.rb | 34 ++++++++++++++++++++++++++++------ 2 files changed, 59 insertions(+), 9 deletions(-) diff --git a/lib/make.rb b/lib/make.rb index e36943f..1609f01 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -28,7 +28,7 @@ module Make class Context attr_accessor :name, :use_cache, :environment, :steps, :packages, :repository - attr_accessor :target + attr_accessor :target, :archive, :source_type def initialize(options: OpenStruct.new) @target = options.target || :user @@ -52,7 +52,21 @@ module Make # puts makefile @packages = makefile["packages"] || [] + + project_sources = { + repository: makefile["repository"] != nil, + archive: makefile["archive"] != nil + }.select { |_, v| v } + + if project_sources.size == 1 + @source_type = project_sources.keys.first + else + raise "Exactly one type of source: 'repository' or 'archive'" + end + puts @source_type + @repository = Make.rostruct(makefile["repository"] || OpenStruct.new) + @archive = Make.rostruct(makefile["archive"] || OpenStruct.new) @steps = makefile["steps"] || [] @environment = ENV.to_h.merge( @@ -147,7 +161,14 @@ module Make install @context.rpd do | path | @cwd = path - checkout + + case @context.source_type + when :repository + checkout + when :archive + download_and_extract + end + puts "path: #{path}" Dir.chdir(path) @@ -160,6 +181,13 @@ module Make end end end + + def download_and_extract + archive = @context.archive + + puts archive + exit -1 + end def checkout repo_path = @context.local_repo @@ -195,4 +223,4 @@ module Make builder = Builder.new(context) builder.build end -end \ No newline at end of file +end diff --git a/recipes/nginx/generate.rb b/recipes/nginx/generate.rb index 6f51fd4..cb9f0cc 100644 --- a/recipes/nginx/generate.rb +++ b/recipes/nginx/generate.rb @@ -3,27 +3,47 @@ require 'erb' class NGINXProxy class << self - attr_accessor :domain, :port, :service + attr_accessor :domain, :port, :service, :user def domain(value = nil) @domain = value unless value.nil? @domain end + def user(value = nil) + @user = value unless value.nil? + @user + end + def port(value = nil) @port = value unless value.nil? @port end - + + # This is name of the daemon that will be installed + # This is also master name user as user if not provided def service(value = nil) @service = value unless value.nil? @service end def generate - template = File.read("proxy.erb") - template = ERB.new(template) - template.result(binding) + template = File.read("proxy.erb") + template = ERB.new(template) + template.result(binding) + end + + def available_path + "/etc/nginx/sites-available/#{service}.#{domain}" + end + + def enabled_path + "/etc/nginx/sites-available/#{service}.#{domain}" + end + + def install + config = generate + File.write end end end @@ -31,7 +51,9 @@ end class ExampleProxy < NGINXProxy domain "gurgul.org" service "forgejo" + user "git" port 3000 end -puts ExampleProxy.generate \ No newline at end of file +puts ExampleProxy.generate +puts ExampleProxy.path From 3013fda863b0a680455bb87485ffa0a0d71a5fc3 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 4 Aug 2025 19:43:28 +0200 Subject: [PATCH 05/52] Add download manager --- lib/archive.rb | 93 ++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 93 insertions(+) create mode 100644 lib/archive.rb diff --git a/lib/archive.rb b/lib/archive.rb new file mode 100644 index 0000000..d8fde1b --- /dev/null +++ b/lib/archive.rb @@ -0,0 +1,93 @@ +require 'net/http' +require 'uri' +require 'open3' +require 'fileutils' +require 'open-uri' + +module Archive + MAGIC_NUMBERS = { + "\x1F\x8B" => 'gzip', + "\x50\x4B\x03\x04" => 'zip', + "\x28\xB5\x2F\xFD" => 'zstd', + "\xFD\x37\x7A\x58" => 'xz' + } + + def self.detect_format_from_url(url) + case File.extname(url) + when '.gz', '.tgz' then 'gzip' + when '.zip' then 'zip' + when '.zst' then 'zstd' + when '.xz' then 'xz' + else nil + end + end + + def self.detect_format_from_headers(uri) + response = Net::HTTP.get_response(uri) + content_type = response['content-type'] + case content_type + when /gzip/ then 'gzip' + when /zip/ then 'zip' + when /zstd/ then 'zstd' + when /xz/ then 'xz' + else nil + end + end + + def self.detect_format_from_magic(file_path) + File.open(file_path, 'rb') do |f| + bytes = f.read(4) + MAGIC_NUMBERS.each do |magic, format| + return format if bytes.start_with?(magic) + end + end + nil + end + + def self.download_file(uri, output_path) + URI.open(uri) do |input| + File.open(output_path, 'wb') do |output| + IO.copy_stream(input, output) + end + end + end + + def self.extract_archive(file_path, format) + case format + when 'gzip' + system("tar -xzf #{Shellwords.escape(file_path)}") + when 'zip' + system("unzip #{Shellwords.escape(file_path)}") + when 'zstd' + system("unzstd #{Shellwords.escape(file_path)}") + when 'xz' + system("tar -xJf #{Shellwords.escape(file_path)}") + else + raise "Unsupported archive format: #{format}" + end + end + + def self.fetch_and_extract(url) + uri = URI.parse(url) + file_name = File.basename(uri.path) + tmp_path = "/tmp/#{file_name}" + + format = detect_format_from_url(url) + puts "format #{format}" + download_file(uri, tmp_path) + + unless format + format = detect_format_from_headers(uri) + end + + unless format + format = detect_format_from_magic(tmp_path) + end + + raise "Could not determine archive format" unless format + + extract_archive(tmp_path, format) + end +end + # Example usage: + # fetch_and_extract("https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz") From f3bb91d09e0387a34ff60cd60b61c6cccd513664 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 5 Aug 2025 10:48:11 +0200 Subject: [PATCH 06/52] save --- lib/archive.rb | 56 +++++++++++++++++++++++++++++--------------------- 1 file changed, 33 insertions(+), 23 deletions(-) diff --git a/lib/archive.rb b/lib/archive.rb index d8fde1b..3f71694 100644 --- a/lib/archive.rb +++ b/lib/archive.rb @@ -3,6 +3,7 @@ require 'uri' require 'open3' require 'fileutils' require 'open-uri' +require 'shellwords' module Archive MAGIC_NUMBERS = { @@ -12,6 +13,32 @@ module Archive "\xFD\x37\x7A\x58" => 'xz' } + def self.fetch_and_extract(url, cache_path, destination_dir) + uri = URI.parse(url) + file_name = File.basename(uri.path) + archive_path = File.join(destination_dir, file_name) + + format = detect_format_from_url(url) + puts "format #{format}" + download_file(uri, archive_path) + + unless format + puts "case 1" + format = detect_format_from_headers(uri) + end + + unless format + puts "case 2" + format = detect_format_from_magic(archive_path) + end + + raise "Could not determine archive format" unless format + + extract_archive(archive_path, format) + end + + private + def self.detect_format_from_url(url) case File.extname(url) when '.gz', '.tgz' then 'gzip' @@ -61,33 +88,16 @@ module Archive when 'zstd' system("unzstd #{Shellwords.escape(file_path)}") when 'xz' - system("tar -xJf #{Shellwords.escape(file_path)}") + # if file_path.end_with?('.tar.xz') + if `file #{Shellwords.escape(file_path)}`.include?('tar archive') + system("tar -xJf #{Shellwords.escape(file_path)}") + else + system("xz -dk #{Shellwords.escape(file_path)}") + end else raise "Unsupported archive format: #{format}" end end - - def self.fetch_and_extract(url) - uri = URI.parse(url) - file_name = File.basename(uri.path) - tmp_path = "/tmp/#{file_name}" - - format = detect_format_from_url(url) - puts "format #{format}" - download_file(uri, tmp_path) - - unless format - format = detect_format_from_headers(uri) - end - - unless format - format = detect_format_from_magic(tmp_path) - end - - raise "Could not determine archive format" unless format - - extract_archive(tmp_path, format) - end end # Example usage: # fetch_and_extract("https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz") From 7ab7d5529245ebe803c2a9b381e5cd8145913fe5 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 5 Aug 2025 18:09:06 +0200 Subject: [PATCH 07/52] save --- recipes/forgejo-recipe/debian.yml | 6 ++++++ 1 file changed, 6 insertions(+) create mode 100644 recipes/forgejo-recipe/debian.yml diff --git a/recipes/forgejo-recipe/debian.yml b/recipes/forgejo-recipe/debian.yml new file mode 100644 index 0000000..fb41220 --- /dev/null +++ b/recipes/forgejo-recipe/debian.yml @@ -0,0 +1,6 @@ + +archive: + url: https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz + +steps: + - echo "ls" From e515e5a76c3a4ce34e7a205d7807a666941bc903 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 6 Aug 2025 07:17:19 +0200 Subject: [PATCH 08/52] save --- lib/make.rb | 17 ++++++++++++++--- lib/system/debian.rb | 7 +++++++ lib/system/macos.rb | 7 +++++++ recipes/dry-run.yml | 2 +- 4 files changed, 29 insertions(+), 4 deletions(-) diff --git a/lib/make.rb b/lib/make.rb index be1d190..e36943f 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -56,7 +56,9 @@ module Make @steps = makefile["steps"] || [] @environment = ENV.to_h.merge( - "PREFIX" => get_prefix + "PREFIX" => get_prefix, + "CPUS" => System.cpus.to_s, + "SUDO" => (@target == :user ? "" : "sudo") ) environment = makefile["environment"] @@ -69,13 +71,22 @@ module Make end def get_prefix + sys_prefix = case System.os_name + when :macos + "/usr" + else + "" + end + case @target when :user "#{ENV["HOME"]}/.local" when :package - "/pkg/#{@name}/#{@repository.branch}" + "#{sys_prefix}/pkg/#{@name}/#{@repository.branch}" when :system - "/" + "#{sys_prefix}/" + else + raise "Unknown target: #{@target.inspect}" end end diff --git a/lib/system/debian.rb b/lib/system/debian.rb index 61170c8..6504a2f 100644 --- a/lib/system/debian.rb +++ b/lib/system/debian.rb @@ -1,8 +1,15 @@ +require 'open3' + module DebianSystem def os_name "Debian Linux" end + def cpus + stdout, stderr, status = Open3.capture3("nproc") + stdout.strip.to_i + end + def install(packages) missing_packages = packages.reject { |pkg| package_installed?(pkg) } diff --git a/lib/system/macos.rb b/lib/system/macos.rb index 7cacd3d..b2a6a22 100644 --- a/lib/system/macos.rb +++ b/lib/system/macos.rb @@ -1,8 +1,15 @@ +require 'open3' + module MacOSSystem def os_name "macOS" end + def cpus + stdout, stderr, status = Open3.capture3("sysctl -n hw.ncpu") + stdout.strip.to_i + end + def install(packages) missing_packages = packages.reject { |pkg| package_installed?(pkg) } diff --git a/recipes/dry-run.yml b/recipes/dry-run.yml index f5258b5..77e8509 100644 --- a/recipes/dry-run.yml +++ b/recipes/dry-run.yml @@ -9,7 +9,7 @@ repository: branch: main steps: - - echo installing in $PREFIX + - echo installing in $PREFIX and cpus $CPUS - cd $WORK_DIR - ls From f7d2665d4f439603994b03b324bc602b2623467b Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 6 Aug 2025 08:00:01 +0200 Subject: [PATCH 09/52] save --- .../data/templates}/nginx/default.erb | 0 .../data/templates}/nginx/proxy.erb | 0 lib/make.rb | 1 + recipes/nginx/generate.rb => lib/nginx.rb | 21 +++++++++---------- lib/user.rb | 5 +++++ recipes/example-roda.rb | 11 ++++++++++ 6 files changed, 27 insertions(+), 11 deletions(-) rename {recipes => lib/data/templates}/nginx/default.erb (100%) rename {recipes => lib/data/templates}/nginx/proxy.erb (100%) rename recipes/nginx/generate.rb => lib/nginx.rb (71%) create mode 100644 lib/user.rb create mode 100644 recipes/example-roda.rb diff --git a/recipes/nginx/default.erb b/lib/data/templates/nginx/default.erb similarity index 100% rename from recipes/nginx/default.erb rename to lib/data/templates/nginx/default.erb diff --git a/recipes/nginx/proxy.erb b/lib/data/templates/nginx/proxy.erb similarity index 100% rename from recipes/nginx/proxy.erb rename to lib/data/templates/nginx/proxy.erb diff --git a/lib/make.rb b/lib/make.rb index 1609f01..b8330b1 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -7,6 +7,7 @@ require 'fileutils' require 'open3' # make for: the user, system, package +# user/usr, system/sys, package/pkg # as regular user, if dependencies provided # user: $HOME/.local diff --git a/recipes/nginx/generate.rb b/lib/nginx.rb similarity index 71% rename from recipes/nginx/generate.rb rename to lib/nginx.rb index cb9f0cc..731af57 100644 --- a/recipes/nginx/generate.rb +++ b/lib/nginx.rb @@ -3,7 +3,12 @@ require 'erb' class NGINXProxy class << self - attr_accessor :domain, :port, :service, :user + attr_accessor :domain, :port, :service, :user, :willcard + + def willcard(value = nil) + @willcard = value unless value.nil? + @willcard + end def domain(value = nil) @domain = value unless value.nil? @@ -27,8 +32,11 @@ class NGINXProxy @service end + ## bundle exec rackup -s puma -b unix:///run/user/1000/http.sock + def generate - template = File.read("proxy.erb") + template_path = File.join(__dir__, 'data', 'templates', 'nginx', 'proxy.erb') + template = File.read(template_path) template = ERB.new(template) template.result(binding) end @@ -48,12 +56,3 @@ class NGINXProxy end end -class ExampleProxy < NGINXProxy - domain "gurgul.org" - service "forgejo" - user "git" - port 3000 -end - -puts ExampleProxy.generate -puts ExampleProxy.path diff --git a/lib/user.rb b/lib/user.rb new file mode 100644 index 0000000..0e36e36 --- /dev/null +++ b/lib/user.rb @@ -0,0 +1,5 @@ + +module User + + +end \ No newline at end of file diff --git a/recipes/example-roda.rb b/recipes/example-roda.rb new file mode 100644 index 0000000..d721902 --- /dev/null +++ b/recipes/example-roda.rb @@ -0,0 +1,11 @@ +require 'nginx' + +class ExampleProxy < NGINXProxy + domain "gurgul.org" + service "forgejo" + user "git" + port 3000 +end + +puts ExampleProxy.generate +puts ExampleProxy.available_path From 045864495f0d57796cb0c7d482e533b6f56675e5 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 6 Aug 2025 08:16:56 +0200 Subject: [PATCH 10/52] update url --- install | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/install b/install index 3c5243c..3be7421 100755 --- a/install +++ b/install @@ -28,7 +28,7 @@ if [ "$system" = true ] && [ "$EUID" -ne 0 ]; then exec sudo INSTALL_HOME="$INSTALL_HOME" "$0" "$@" fi -REPO_URL="https://gitlab.com/artur.gurgul/home.git" +REPO_URL="https://gurgul.pro/artur/environment.git" is_debian_like() { if [ -r /etc/os-release ]; then From 757921be5b2a6e465377ed150fa4e70c723301f3 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Wed, 6 Aug 2025 10:17:48 +0200 Subject: [PATCH 11/52] update --- bin/dat | 1 + lib/archive.rb | 104 ++++++++++++++++++++++-------- lib/make.rb | 16 ++++- recipes/forgejo-recipe/debian.yml | 2 +- 4 files changed, 94 insertions(+), 29 deletions(-) diff --git a/bin/dat b/bin/dat index ce39498..b6fd0fb 100755 --- a/bin/dat +++ b/bin/dat @@ -12,6 +12,7 @@ options = OpenStruct.new # take the first so OptionParser will not see it subcommand = ARGV.shift&.to_sym +options.name = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil OptionParser.new do |opt| opt.on('-i', '--install', 'Install dat for the user') { |o| options.type = :install } diff --git a/lib/archive.rb b/lib/archive.rb index 3f71694..6729d76 100644 --- a/lib/archive.rb +++ b/lib/archive.rb @@ -13,7 +13,17 @@ module Archive "\xFD\x37\x7A\x58" => 'xz' } - def self.fetch_and_extract(url, cache_path, destination_dir) + def self.extract(archive_path, destination_dir) + format = detect_format_from_url(archive_path) + + unless format + format = detect_format_from_magic(archive_path) + end + + extract_archive(archive_path, destination_dir, format) + end + + def self.fetch_and_extract(url, cache_path, destination_dir) uri = URI.parse(url) file_name = File.basename(uri.path) archive_path = File.join(destination_dir, file_name) @@ -21,7 +31,8 @@ module Archive format = detect_format_from_url(url) puts "format #{format}" download_file(uri, archive_path) - + + # Probably do not work unless format puts "case 1" format = detect_format_from_headers(uri) @@ -37,6 +48,20 @@ module Archive extract_archive(archive_path, format) end + def self.download_file(uri, output_path, forced=false) + return if File.exist?(output_path) && !forced + + # Ensure the directory exists + dir = File.dirname(output_path) + FileUtils.mkdir_p(dir) unless Dir.exist?(dir) + + URI.open(uri) do |input| + File.open(output_path, 'wb') do |output| + IO.copy_stream(input, output) + end + end + end + private def self.detect_format_from_url(url) @@ -71,33 +96,60 @@ module Archive nil end - def self.download_file(uri, output_path) - URI.open(uri) do |input| - File.open(output_path, 'wb') do |output| - IO.copy_stream(input, output) - end + + require 'shellwords' + + def self.extract_archive(file_path, destination_dir, format) + # TODO: need to be checked + escaped_file = Shellwords.escape(file_path) + escaped_dir = Shellwords.escape(destination_dir) + + system("mkdir -p #{escaped_dir}") + + case format + when 'gzip' + system("tar -xzf #{escaped_file} -C #{escaped_dir}") + when 'zip' + system("unzip -d #{escaped_dir} #{escaped_file}") + when 'zstd' + if `file #{escaped_file}`.include?('tar archive') + system("unzstd -c #{escaped_file} | tar -xf - -C #{escaped_dir}") + else + system("unzstd -o \"#{escaped_dir}\" #{escaped_file}") end + when 'xz' + if `file #{escaped_file}`.include?('tar archive') + system("tar -xJf #{escaped_file} -C #{escaped_dir}") + else + system("xz -dk #{escaped_file}") + decompressed = file_path.sub(/\.xz$/, '') + system("mv #{Shellwords.escape(decompressed)} #{escaped_dir}/") + end + else + raise "Unsupported archive format: #{format}" + end end - def self.extract_archive(file_path, format) - case format - when 'gzip' - system("tar -xzf #{Shellwords.escape(file_path)}") - when 'zip' - system("unzip #{Shellwords.escape(file_path)}") - when 'zstd' - system("unzstd #{Shellwords.escape(file_path)}") - when 'xz' - # if file_path.end_with?('.tar.xz') - if `file #{Shellwords.escape(file_path)}`.include?('tar archive') - system("tar -xJf #{Shellwords.escape(file_path)}") - else - system("xz -dk #{Shellwords.escape(file_path)}") - end - else - raise "Unsupported archive format: #{format}" - end - end + + # def self.extract_archive(file_path, destination_dir, format) + # case format + # when 'gzip' + # system("tar -xzf #{Shellwords.escape(file_path)}") + # when 'zip' + # system("unzip #{Shellwords.escape(file_path)}") + # when 'zstd' + # system("unzstd #{Shellwords.escape(file_path)}") + # when 'xz' + # # if file_path.end_with?('.tar.xz') + # if `file #{Shellwords.escape(file_path)}`.include?('tar archive') + # system("tar -xJf #{Shellwords.escape(file_path)}") + # else + # system("xz -dk #{Shellwords.escape(file_path)}") + # end + # else + # raise "Unsupported archive format: #{format}" + # end + # end end # Example usage: # fetch_and_extract("https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz") diff --git a/lib/make.rb b/lib/make.rb index b8330b1..3a324d7 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -123,6 +123,12 @@ module Make "#{ENV["HOME"]}/.cache/dat/repo/#{@name}.git" end + def download_path + require 'addressable/uri' + uri = Addressable::URI.parse(archive.url) + File.join("#{ENV["HOME"]}/.cache/dat/downloads/", uri.domain, uri.path) + end + def to_s vars = instance_variables.map do |var| "#{var.to_s.delete('@')}: #{instance_variable_get(var).inspect}" @@ -184,10 +190,16 @@ module Make end def download_and_extract - archive = @context.archive + # gem install addressable + + archive = @context.archive + puts @context.download_path puts archive - exit -1 + + require 'archive' + Archive.download_file(@context.archive.url, @context.download_path) + Archive.extract(@context.download_path, @cwd) end def checkout diff --git a/recipes/forgejo-recipe/debian.yml b/recipes/forgejo-recipe/debian.yml index fb41220..0159ae8 100644 --- a/recipes/forgejo-recipe/debian.yml +++ b/recipes/forgejo-recipe/debian.yml @@ -3,4 +3,4 @@ archive: url: https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz steps: - - echo "ls" + - mv forgejo-12.0.1-linux-amd64 ~/.local/bin/forgejo From 953985a3eaa9555f245d82c1a70f2bd48b15470d Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Wed, 6 Aug 2025 10:45:12 +0200 Subject: [PATCH 12/52] update vim config --- home/.config/nvim/lua/lsp.lua | 7 +++++++ install | 4 ++++ 2 files changed, 11 insertions(+) diff --git a/home/.config/nvim/lua/lsp.lua b/home/.config/nvim/lua/lsp.lua index 1352071..eadba45 100644 --- a/home/.config/nvim/lua/lsp.lua +++ b/home/.config/nvim/lua/lsp.lua @@ -16,6 +16,13 @@ lspconfig.gopls.setup { }, } +-- ruby LSP +lspconfig.ruby_lsp.setup({ + cmd = { "ruby-lsp" }, -- or provide full path if needed + filetypes = { "ruby" }, + root_dir = lspconfig.util.root_pattern("Gemfile", ".git"), +}) + -- Completion local cmp = require("cmp") cmp.setup({ diff --git a/install b/install index 3be7421..1b8b6c8 100755 --- a/install +++ b/install @@ -1,5 +1,9 @@ #!/usr/bin/env bash +# TODO: Packages that need to be added +# gem install ruby-lsp + +# # ./install --system --forced # --forced (not forced, does not delete old version, just ends in failure) # --system (refault user) From 2578cd7a6d4c516319c4b0d6881b4c67fd6652b1 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Wed, 6 Aug 2025 16:50:03 +0200 Subject: [PATCH 13/52] add postgresql --- home/.config/nvim/lua/plugins.lua | 15 +++++- lib/data/templates/nginx/proxy.erb | 2 +- lib/data/templates/postgresql/pg_hba.conf.erb | 2 + .../templates/postgresql/postgresql.conf.erb | 52 +++++++++++++++++++ lib/data/templates/postgresql/test.erb | 4 ++ lib/setup/postgresql.rb | 23 ++++++++ lib/templates.rb | 20 +++++++ recipes/forgejo-recipe/debian.yml | 3 ++ recipes/postgresql/debian-setup.yml | 24 +++++++++ recipes/postgresql/debian.yml | 21 ++++++++ 10 files changed, 164 insertions(+), 2 deletions(-) create mode 100644 lib/data/templates/postgresql/pg_hba.conf.erb create mode 100644 lib/data/templates/postgresql/postgresql.conf.erb create mode 100644 lib/data/templates/postgresql/test.erb create mode 100644 lib/setup/postgresql.rb create mode 100644 lib/templates.rb create mode 100644 recipes/postgresql/debian-setup.yml create mode 100644 recipes/postgresql/debian.yml diff --git a/home/.config/nvim/lua/plugins.lua b/home/.config/nvim/lua/plugins.lua index 8ecfd76..fe363a7 100644 --- a/home/.config/nvim/lua/plugins.lua +++ b/home/.config/nvim/lua/plugins.lua @@ -33,7 +33,20 @@ require("lazy").setup({ { "hrsh7th/vim-vsnip" }, -- Treesitter - { "nvim-treesitter/nvim-treesitter", build = ":TSUpdate" }, + { + "nvim-treesitter/nvim-treesitter", + build = ":TSUpdate", + config = function() + require("nvim-treesitter.configs").setup({ + ensure_installed = { "ruby", "lua", "vim", "bash", "json" }, -- include ruby + highlight = { + enable = true, + additional_vim_regex_highlighting = false, + }, + }) + end + }, + -- Appearance { "tomasiser/vim-code-dark" }, diff --git a/lib/data/templates/nginx/proxy.erb b/lib/data/templates/nginx/proxy.erb index 81c813b..061b96a 100644 --- a/lib/data/templates/nginx/proxy.erb +++ b/lib/data/templates/nginx/proxy.erb @@ -26,4 +26,4 @@ server { ssl_certificate_key /etc/letsencrypt/live/<%= domain %>/privkey.pem; include /etc/letsencrypt/options-ssl-nginx.conf; ssl_dhparam /etc/letsencrypt/ssl-dhparams.pem; -} \ No newline at end of file +} diff --git a/lib/data/templates/postgresql/pg_hba.conf.erb b/lib/data/templates/postgresql/pg_hba.conf.erb new file mode 100644 index 0000000..1afb456 --- /dev/null +++ b/lib/data/templates/postgresql/pg_hba.conf.erb @@ -0,0 +1,2 @@ +local all all peer +hostssl all all 0.0.0.0/0 scram-sha-256 diff --git a/lib/data/templates/postgresql/postgresql.conf.erb b/lib/data/templates/postgresql/postgresql.conf.erb new file mode 100644 index 0000000..c0617d1 --- /dev/null +++ b/lib/data/templates/postgresql/postgresql.conf.erb @@ -0,0 +1,52 @@ + +data_directory = '/var/lib/postgresql/15/main' +hba_file = '/etc/postgresql/15/main/pg_hba.conf' +ident_file = '/etc/postgresql/15/main/pg_ident.conf' + + +#listen_addresses = 'localhost' +listen_addresses = '*' +port = 5432 +max_connections = 100 +unix_socket_directories = '/var/run/postgresql' +password_encryption = scram-sha-256 + +ssl = on +#ssl_ca_file = '' +ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' +#ssl_crl_file = '' +#ssl_crl_dir = '' +ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' +#ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers +#ssl_prefer_server_ciphers = on +#ssl_ecdh_curve = 'prime256v1' +#ssl_min_protocol_version = 'TLSv1.2' +#ssl_max_protocol_version = '' +#ssl_dh_params_file = '' +#ssl_passphrase_command = '' +#ssl_passphrase_command_supports_reload = off + +shared_buffers = 128MB +dynamic_shared_memory_type = posix # the default is usually the first option + +max_wal_size = 1GB +min_wal_size = 80MB + +log_line_prefix = '%m [%p] %q%u@%d ' +log_timezone = 'Etc/UTC' + + +cluster_name = '15/main' + +datestyle = 'iso, mdy' +timezone = 'Etc/UTC' +lc_messages = 'C.UTF-8' +lc_monetary = 'C.UTF-8' +lc_numeric = 'C.UTF-8' +lc_time = 'C.UTF-8' + +default_text_search_config = 'pg_catalog.english' + +# include files ending in '.conf' from +include_dir = 'conf.d' + diff --git a/lib/data/templates/postgresql/test.erb b/lib/data/templates/postgresql/test.erb new file mode 100644 index 0000000..b0a5afa --- /dev/null +++ b/lib/data/templates/postgresql/test.erb @@ -0,0 +1,4 @@ + + + +<%= te %> diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb new file mode 100644 index 0000000..0b12a8a --- /dev/null +++ b/lib/setup/postgresql.rb @@ -0,0 +1,23 @@ + +module Setup + require_relative '../templates' + + module PostgreSQL + extend Templates + + # attr_accessor :te + + + + def self.make_config + pg_hba = render("pg_hba.conf") + + test = render("test", te: "This is a test string") + puts test + end + + def self.install + make_config + end + end +end diff --git a/lib/templates.rb b/lib/templates.rb new file mode 100644 index 0000000..a5bd996 --- /dev/null +++ b/lib/templates.rb @@ -0,0 +1,20 @@ +require 'erb' + +module Templates + def render(name, locals = {}) + # caller_module = Module.nesting.first.to_s.split('::').last&.downcase || 'common' + caller_file = caller_locations(1, 1)[0].absolute_path + inferred_dir = File.basename(caller_file).sub(/^install-/, '').sub(/\.rb$/, '') + puts "caller name: #{inferred_dir}" + + template_path = File.join(__dir__, 'data', 'templates', inferred_dir, "#{name}.erb") + template = File.read(template_path) + erb = ERB.new(template) + + # erb.result(binding) + + context = Struct.new(*locals.keys).new(*locals.values) + erb.result(context.instance_eval { binding }) + end +end + diff --git a/recipes/forgejo-recipe/debian.yml b/recipes/forgejo-recipe/debian.yml index 0159ae8..b620dfb 100644 --- a/recipes/forgejo-recipe/debian.yml +++ b/recipes/forgejo-recipe/debian.yml @@ -1,4 +1,7 @@ +service: + name: forgejo + archive: url: https://codeberg.org/forgejo/forgejo/releases/download/v12.0.1/forgejo-12.0.1-linux-amd64.xz diff --git a/recipes/postgresql/debian-setup.yml b/recipes/postgresql/debian-setup.yml new file mode 100644 index 0000000..126cc9f --- /dev/null +++ b/recipes/postgresql/debian-setup.yml @@ -0,0 +1,24 @@ +environment: + PG_DOMAIN: gurgul.org + + +packages: + - postgresql + - postgresql-contrib + +steps: + - $SUDO systemctl enable postgresql + - $SUDO systemctl start postgresql + # Installing certificates + - $SUDO mkdir /etc/postgresql/ssl + - $SUDO cp /etc/letsencrypt/live/$PG_DOMAIN/fullchain.pem /etc/postgresql/ssl/server.crt + - $SUDO cp /etc/letsencrypt/live/$PG_DOMAIN/privkey.pem /etc/postgresql/ssl/server.key + - $SUDO chown postgres:postgres /etc/postgresql/ssl/server.* + - $SUDO chmod 600 /etc/postgresql/ssl/server.key + + +actions: + # dat action postgresql:add-user -u user + add-user: + - sudo -u postgres createuser --no-superuser --no-createdb --no-createrole $DB_USER + - sudo -u postgres createdb -O $DB_USER $DB_USER diff --git a/recipes/postgresql/debian.yml b/recipes/postgresql/debian.yml new file mode 100644 index 0000000..1d0e951 --- /dev/null +++ b/recipes/postgresql/debian.yml @@ -0,0 +1,21 @@ + +packages: + - build-essential + - libreadline-dev + - zlib1g-dev + - flex + - bison + - libssl-dev + - libxml2-dev + - libxslt1-dev + - libpam0g-dev + - libedit-dev + +repository: + url: https://github.com/postgres/postgres.git + branch: REL_17_5 + +steps: + - ./configure --prefix=$PREFIX + - make -j$CPUS + - make install From 9c77ae9d2c60d4d9fca410d1b8335fbb440ccb9c Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Wed, 6 Aug 2025 17:49:36 +0200 Subject: [PATCH 14/52] make it possible to call DSL script --- bin/dat | 4 +++ lib/execute.rb | 71 ++++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+) create mode 100644 lib/execute.rb diff --git a/bin/dat b/bin/dat index b6fd0fb..274d5fa 100755 --- a/bin/dat +++ b/bin/dat @@ -56,6 +56,10 @@ when :make puts "making..." require 'make' Make.command(options) +when :execute + puts "Ececuting: #{options.name}" + require 'execute' + Execute.file(options) when :goto Dir.chdir(ENV["DAT_ROOT"]) when :vm diff --git a/lib/execute.rb b/lib/execute.rb new file mode 100644 index 0000000..d91d6ef --- /dev/null +++ b/lib/execute.rb @@ -0,0 +1,71 @@ + +module Execute + + +end + + +require 'rubygems' +require 'ostruct' + +# dat execute progress +# dat execute < default Configfile + +module Execute + def dependency(name) + puts "Checking for #{name}..." + + unless gem_installed?(name) + puts "Installing #{name}..." + system("gem install #{name}") + else + puts "#{name} is already installed." + end + + require name + rescue LoadError => e + puts "Failed to load #{name}: #{e.message}" + end + + def gem_installed?(name) + Gem::Specification::find_all_by_name(name).any? + end + + def postgres + require 'setup/postgresql' + puts 'installing postgres' + Setup::PostgreSQL.install + end + + def service(name) + if block_given? + context = OpenStruct.new + output = yield context + puts output + puts "context: #{context}" + else + puts "No block provided" + end + end + + def user(name) + puts "Creating user: #{name}" + end + + def install(*packages) + packages.each do |pkg| + puts "Installing #{pkg}" + end + end + + def self.file(options) + puts "options: #{options}" + dsl = Object.new + dsl.extend(Execute) + # Configfile + path = File.join(Dir.pwd, options.name || "./Configfile") + dsl.instance_eval(File.read(path), path) + # Execute the config + end +end + From a7807f089cf4c661fa39a303162fa2b3e65fc8c1 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Wed, 6 Aug 2025 17:55:14 +0200 Subject: [PATCH 15/52] add doc --- doc/execute.md | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) create mode 100644 doc/execute.md diff --git a/doc/execute.md b/doc/execute.md new file mode 100644 index 0000000..efa8ace --- /dev/null +++ b/doc/execute.md @@ -0,0 +1,37 @@ + + +It is how you can describe config of your server + + + +```ruby +user :artur +user :lucyna + +service :forgejo do |context| + context.home_page = true + # per default all users are allowed + context.users = [:artur] +end + +service :zulip + +# this should be already installed, becouse it is required by others +service :postgresql + +install :nvim, :python +``` + + +And then save it as `Configfile` as default or with specyfic name + + +```bash +# if default file name +dat execute + +# if non standard file +dat execute file-name + +``` + From ed7fc9aa7fcea98c1418d3e16b37b2785f6a1fc5 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 13:02:01 +0200 Subject: [PATCH 16/52] add possibility to create package --- bin/dat | 8 +++ .../templates/postgresql/postgresql.service | 29 +++++++++++ lib/execute.rb | 6 +++ lib/make.rb | 52 ++++++++++++++++--- lib/setup/postgresql.rb | 15 +++++- recipes/postgresql/debian.yml | 4 +- 6 files changed, 104 insertions(+), 10 deletions(-) create mode 100644 lib/data/templates/postgresql/postgresql.service diff --git a/bin/dat b/bin/dat index 274d5fa..0fdd07a 100755 --- a/bin/dat +++ b/bin/dat @@ -56,6 +56,14 @@ when :make puts "making..." require 'make' Make.command(options) +when :pkgmake + puts "Making package #{options.name}" + require 'make' + Make.create_package(options) +when :pkginstall + require 'make' + # TODO: wrong naming it is not name, but path + Make.install_package(options.name) when :execute puts "Ececuting: #{options.name}" require 'execute' diff --git a/lib/data/templates/postgresql/postgresql.service b/lib/data/templates/postgresql/postgresql.service new file mode 100644 index 0000000..3976e51 --- /dev/null +++ b/lib/data/templates/postgresql/postgresql.service @@ -0,0 +1,29 @@ +[Unit] +Description=PostgreSQL <%= version %> database server +After=network.target + +[Service] +Type=notify + +User=postgres +Group=services + +ExecStart=<%= prefix %>/bin/postgres -D <%= database_path %>/data +ExecReload=/bin/kill -HUP $MAINPID +KillMode=mixed +TimeoutSec=300 +Restart=on-failure +NotifyAccess=all + +# Security +ProtectSystem=full +ProtectHome=true +PrivateTmp=true +NoNewPrivileges=true + +# Resource Limits +LimitNOFILE=65536 + +[Install] +WantedBy=multi-user.target + diff --git a/lib/execute.rb b/lib/execute.rb index d91d6ef..fd495d4 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -10,6 +10,9 @@ require 'ostruct' # dat execute progress # dat execute < default Configfile +# dat execute --dry-run +# Execute single command +# dat command service zulip module Execute def dependency(name) @@ -59,6 +62,9 @@ module Execute end def self.file(options) + require 'make' + puts Make.context("postgresql") + puts "options: #{options}" dsl = Object.new dsl.extend(Execute) diff --git a/lib/make.rb b/lib/make.rb index 3a324d7..e5ba949 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -29,7 +29,7 @@ module Make class Context attr_accessor :name, :use_cache, :environment, :steps, :packages, :repository - attr_accessor :target, :archive, :source_type + attr_accessor :target, :archive, :source_type, :version def initialize(options: OpenStruct.new) @target = options.target || :user @@ -55,15 +55,21 @@ module Make @packages = makefile["packages"] || [] project_sources = { - repository: makefile["repository"] != nil, - archive: makefile["archive"] != nil + repository: makefile["repository"], + archive: makefile["archive"] }.select { |_, v| v } if project_sources.size == 1 + source_data = project_sources.values.first @source_type = project_sources.keys.first + @version = source_data["version"] || source_data["branch"] + + raise "No version provided" if @version == nil + @version = @version.to_s else raise "Exactly one type of source: 'repository' or 'archive'" end + puts @source_type @repository = Make.rostruct(makefile["repository"] || OpenStruct.new) @@ -97,7 +103,7 @@ module Make when :user "#{ENV["HOME"]}/.local" when :package - "#{sys_prefix}/pkg/#{@name}/#{@repository.branch}" + "#{sys_prefix}/pkg/#{@name}/#{@version}" when :system "#{sys_prefix}/" else @@ -227,13 +233,45 @@ module Make def install System.install(context.packages) end + + def create_package(options) + puts "Creating packahge for options: #{options}" + puts @context + puts "Prefix #{@context.get_prefix}" + system("tar --zstd -Pcf #{@context.name}-#{@context.version}.pkg #{@context.get_prefix}") + end end # dat make -t pkg --cache --name dry-run # dat make --name dry-run def self.command(options) - context = Context.new(options: options) - builder = Builder.new(context) - builder.build + context = Context.new(options: options) + builder = Builder.new(context) + builder.build end + + # dat pkginstall {file-path} + # an example `dat pkginstall postgresql-17.5.pkg` + def self.install_package(path) + puts "Installing package: #{path}" + system("sudo tar --zstd -xf #{path} -C /") + end + + # dat pkgmake {name} + def self.create_package(options) + options["target"] = :package if options["target"] == nil + context = Context.new(options: options) + builder = Builder.new(context) + builder.create_package(options) + end + + # TODO: better name, it is just to obtain information + # about the package + def self.context(package_name, target = :package) + options = OpenStruct.new + options["target"] = target + options["name"] = package_name + Context.new(options: options) + end + end diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index 0b12a8a..98652b1 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -5,14 +5,25 @@ module Setup module PostgreSQL extend Templates + + # attr_accessor :te - + def self.init_db + system(" -D /opt/pgsql/16/data") + end def self.make_config pg_hba = render("pg_hba.conf") - test = render("test", te: "This is a test string") + # test = render("test", te: "This is a test string") + posgresql_conf = render("postgresql") + service = render( + "postgresql.service", + prefix: "/pkg/postgresql/REL_17_5/", + version: "16.5", + data_url: "/data/postgresql/16/" + ) puts test end diff --git a/recipes/postgresql/debian.yml b/recipes/postgresql/debian.yml index 1d0e951..fa0d991 100644 --- a/recipes/postgresql/debian.yml +++ b/recipes/postgresql/debian.yml @@ -14,8 +14,10 @@ packages: repository: url: https://github.com/postgres/postgres.git branch: REL_17_5 + version: 17.5 steps: - ./configure --prefix=$PREFIX - make -j$CPUS - - make install + - $SUDO mkdir -p $PREFIX + - $SUDO make install From 29a9b08573fa0db8b7071d84b04a8283997c7d2d Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 14:23:42 +0200 Subject: [PATCH 17/52] generate systemd for postgres --- ...tgresql.service => postgresql.service.erb} | 2 +- lib/execute.rb | 52 ++++++++++++++++--- lib/setup/postgresql.rb | 31 +++++------ recipes/postgresql/debian.yml | 2 + 4 files changed, 63 insertions(+), 24 deletions(-) rename lib/data/templates/postgresql/{postgresql.service => postgresql.service.erb} (85%) diff --git a/lib/data/templates/postgresql/postgresql.service b/lib/data/templates/postgresql/postgresql.service.erb similarity index 85% rename from lib/data/templates/postgresql/postgresql.service rename to lib/data/templates/postgresql/postgresql.service.erb index 3976e51..3ddaa20 100644 --- a/lib/data/templates/postgresql/postgresql.service +++ b/lib/data/templates/postgresql/postgresql.service.erb @@ -8,7 +8,7 @@ Type=notify User=postgres Group=services -ExecStart=<%= prefix %>/bin/postgres -D <%= database_path %>/data +ExecStart=<%= postgres_bin %> -D <%= database_dir %>/data ExecReload=/bin/kill -HUP $MAINPID KillMode=mixed TimeoutSec=300 diff --git a/lib/execute.rb b/lib/execute.rb index fd495d4..f5c80af 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -15,6 +15,17 @@ require 'ostruct' # dat command service zulip module Execute + class ServiceInstallContext + attr_accessor :bin_dir, :data_dir, :version, :user_name + + def initialize(bin_dir, data_dir, user_name, version) + @bin_dir = bin_dir + @data_dir = data_dir + @user_name = user_name + @version = version + end + end + def dependency(name) puts "Checking for #{name}..." @@ -34,15 +45,44 @@ module Execute Gem::Specification::find_all_by_name(name).any? end - def postgres + def get_install_executor(name) + require 'make' + pdata = Make.context(name) + bin_dir = pdata.get_prefix + data_dir = "/data/#{pdata.name}/#{pdata.version.split(".").first}" + service_install_context = ServiceInstallContext.new( + bin_dir, data_dir, name, pdata.version + ) + + case name + when :postgresql require 'setup/postgresql' - puts 'installing postgres' - Setup::PostgreSQL.install + -> { Setup::PostgreSQL.install(service_install_context) } + # ->(context) { + # Setup::PostgreSQL.install(context) + # } + else + raise "Can't find the executor" + end end + def execute_with_context(context) + + end + # def postgres + # require 'setup/postgresql' + # Setup::PostgreSQL.install + # end + def service(name) + executor = get_install_executor(name) + #executor.call(service_install_context) + executor.call + exit -1 + if block_given? context = OpenStruct.new + # service context output = yield context puts output puts "context: #{context}" @@ -62,16 +102,12 @@ module Execute end def self.file(options) - require 'make' - puts Make.context("postgresql") - - puts "options: #{options}" dsl = Object.new dsl.extend(Execute) + # Configfile path = File.join(Dir.pwd, options.name || "./Configfile") dsl.instance_eval(File.read(path), path) - # Execute the config end end diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index 98652b1..fbbb190 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -1,7 +1,9 @@ module Setup require_relative '../templates' - + # dat execute postgres + # example.rcp + module PostgreSQL extend Templates @@ -9,26 +11,25 @@ module Setup # attr_accessor :te - def self.init_db + def self.init_db(context) system(" -D /opt/pgsql/16/data") end - def self.make_config - pg_hba = render("pg_hba.conf") - - # test = render("test", te: "This is a test string") - posgresql_conf = render("postgresql") + def self.install(context) +# puts context.bin_dir + #make_config +# pg_hba = render("pg_hba.conf") +# +# # test = render("test", te: "This is a test string") +# posgresql_conf = render("postgresql") service = render( "postgresql.service", - prefix: "/pkg/postgresql/REL_17_5/", - version: "16.5", - data_url: "/data/postgresql/16/" + postgres_bin: File.join(context.bin_dir, "/bin/postgres"), + version: context.version, + database_dir: context.data_dir ) - puts test - end - - def self.install - make_config + puts service + end end end diff --git a/recipes/postgresql/debian.yml b/recipes/postgresql/debian.yml index fa0d991..fa81f49 100644 --- a/recipes/postgresql/debian.yml +++ b/recipes/postgresql/debian.yml @@ -12,6 +12,8 @@ packages: - libedit-dev repository: + # Original repository + # https://git.postgresql.org/gitweb/?p=postgresql.git url: https://github.com/postgres/postgres.git branch: REL_17_5 version: 17.5 From 055744e7f74b47b0f199629286b1aa8a21d9be3e Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 16:29:11 +0200 Subject: [PATCH 18/52] calling add user service --- .../postgresql/postgresql.service.erb | 2 +- lib/execute.rb | 30 +++++++++++++++++-- lib/setup/postgresql.rb | 6 +++- lib/user.rb | 5 +++- 4 files changed, 38 insertions(+), 5 deletions(-) diff --git a/lib/data/templates/postgresql/postgresql.service.erb b/lib/data/templates/postgresql/postgresql.service.erb index 3ddaa20..0bc60d9 100644 --- a/lib/data/templates/postgresql/postgresql.service.erb +++ b/lib/data/templates/postgresql/postgresql.service.erb @@ -1,5 +1,5 @@ [Unit] -Description=PostgreSQL <%= version %> database server +Description=PostgreSQL <%= version %> After=network.target [Service] diff --git a/lib/execute.rb b/lib/execute.rb index f5c80af..09e614d 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -25,6 +25,28 @@ module Execute @version = version end end + + class UserInstallContext + attr_accessor :user_name, :type, :can_login + + def initialize(user_name, type = :user, can_login = nil) + @user_name = user_name + @type = type + + if can_login == nil + case type + when :service + @can_login = false + when :user + @can_login = true + else + raise "Can not create user for type: #{type}" + end + else + @can_login = can_login + end + end + end def dependency(name) puts "Checking for #{name}..." @@ -91,8 +113,12 @@ module Execute end end - def user(name) - puts "Creating user: #{name}" + def user(*users) + require 'user' + users.each do |name| + context = UserInstallContext.new(name, :user) + User.install(context) + end end def install(*packages) diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index fbbb190..4e23e57 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -1,6 +1,8 @@ module Setup require_relative '../templates' + require_relative "../user" + require_relative "../execute" # dat execute postgres # example.rcp @@ -28,8 +30,10 @@ module Setup version: context.version, database_dir: context.data_dir ) - puts service + # puts service + user_context = Execute::UserInstallContext.new(context.user_name, :service) + User.install(user_context) end end end diff --git a/lib/user.rb b/lib/user.rb index 0e36e36..230cfbd 100644 --- a/lib/user.rb +++ b/lib/user.rb @@ -1,5 +1,8 @@ module User + def self.install(context) + puts "Creating #{context.type}: #{context.user_name}" + end -end \ No newline at end of file +end From 0f9f1a84f94299b59504448b1743cf51bc47427d Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 16:53:10 +0200 Subject: [PATCH 19/52] implement adding users --- lib/user.rb | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/lib/user.rb b/lib/user.rb index 230cfbd..b084396 100644 --- a/lib/user.rb +++ b/lib/user.rb @@ -1,8 +1,37 @@ module User + require 'system' def self.install(context) puts "Creating #{context.type}: #{context.user_name}" + System.install(["zsh"]) + + user_exists = system("getent passwd #{context.user_name} > /dev/null") + + group = case context.type + when :user + "users" + when :service + "services" + else + rise "Can not create user for unknow type: #{context.type}" + end + + + group_exists = system("getent group #{group} > /dev/null") + unless group_exists + puts "Group '#{group}' does not exist. Creating it..." + system("sudo groupadd #{group}") + end + + if user_exists + puts "User #{context.user_name} already exists. Updating shell and adding to group '#{group}'." + system("sudo usermod -s /usr/bin/zsh #{context.user_name}") + system("sudo usermod -g #{group} #{context.user_name}") + else + puts "User #{context.user_name} does not exist. Creating user..." + system("sudo adduser --disabled-login --gecos \"\" --ingroup #{group} --shell /usr/bin/zsh #{context.user_name}") + end end end From bc0f9db31d42f0d9d5297c372fd5c937b5fd3d41 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 17:36:47 +0200 Subject: [PATCH 20/52] implement adding users --- bin/dat | 4 ++++ lib/execute.rb | 14 +++++++++++--- lib/make.rb | 3 ++- lib/setup/postgresql.rb | 18 +++++++++++++++++- 4 files changed, 34 insertions(+), 5 deletions(-) diff --git a/bin/dat b/bin/dat index 0fdd07a..03e5dc8 100755 --- a/bin/dat +++ b/bin/dat @@ -27,6 +27,9 @@ OptionParser.new do |opt| opt.on('--cache', 'Use cache') do |o| options.use_cache = true end + opt.on('--forced', "if case of existing data, delete and recreate") do |o| + options.forced = true + end opt.on('-t', '--target TARGET', ['user', 'usr', 'package', 'pkg', 'system', 'sys'], 'Target type (user, package, system)') do |target| @@ -67,6 +70,7 @@ when :pkginstall when :execute puts "Ececuting: #{options.name}" require 'execute' + puts "Arguments: #{options}" Execute.file(options) when :goto Dir.chdir(ENV["DAT_ROOT"]) diff --git a/lib/execute.rb b/lib/execute.rb index 09e614d..782885a 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -15,14 +15,21 @@ require 'ostruct' # dat command service zulip module Execute + attr_accessor :options + + def context(new_value = nil) + @options = new_value + end + class ServiceInstallContext - attr_accessor :bin_dir, :data_dir, :version, :user_name + attr_accessor :bin_dir, :data_dir, :version, :user_name, :forced - def initialize(bin_dir, data_dir, user_name, version) + def initialize(bin_dir, data_dir, user_name, version, forced = false) @bin_dir = bin_dir @data_dir = data_dir @user_name = user_name @version = version + @forced = forced end end @@ -73,7 +80,7 @@ module Execute bin_dir = pdata.get_prefix data_dir = "/data/#{pdata.name}/#{pdata.version.split(".").first}" service_install_context = ServiceInstallContext.new( - bin_dir, data_dir, name, pdata.version + bin_dir, data_dir, name, pdata.version, @options.forced ) case name @@ -130,6 +137,7 @@ module Execute def self.file(options) dsl = Object.new dsl.extend(Execute) + dsl.options = options # Configfile path = File.join(Dir.pwd, options.name || "./Configfile") diff --git a/lib/make.rb b/lib/make.rb index e5ba949..b066638 100644 --- a/lib/make.rb +++ b/lib/make.rb @@ -29,13 +29,14 @@ module Make class Context attr_accessor :name, :use_cache, :environment, :steps, :packages, :repository - attr_accessor :target, :archive, :source_type, :version + attr_accessor :target, :archive, :source_type, :version, :forced def initialize(options: OpenStruct.new) @target = options.target || :user @name = options.name @use_cache = options.use_cache || false + @forced = options.forced || false #System.detect_os # rbenv: brew install rbenv && rbenv install 3.0.0 diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index 4e23e57..8feb8db 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -14,7 +14,21 @@ module Setup # attr_accessor :te def self.init_db(context) - system(" -D /opt/pgsql/16/data") + puts "Mode, isForced #{context.forced}" + if Dir.exist?(context.data_dir) + if context.forced + puts("sudo rm -rf #{context.data_dir}") + else + raise "PostgreSQL data already exists" + end + end + system("sudo mkdir -p #{context.data_dir}") + system("sudo chown #{context.user_name}:services #{context.data_dir}") + system("sudo -u postgresql #{File.join(context.bin_dir, "/bin/initdb")} -D #{context.data_dir}") + end + + def self.setup_systemd(context) + end def self.install(context) @@ -34,6 +48,8 @@ module Setup user_context = Execute::UserInstallContext.new(context.user_name, :service) User.install(user_context) + init_db(context) + setup_systemd(context) end end end From 688d23527ff6f91809ddeb1efc769d021fb979be Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 19:41:04 +0200 Subject: [PATCH 21/52] Add setting up systemd --- .../postgresql/postgresql.service.erb | 4 +- lib/execute.rb | 2 + lib/setup/postgresql.rb | 39 +++++++++++++------ 3 files changed, 31 insertions(+), 14 deletions(-) diff --git a/lib/data/templates/postgresql/postgresql.service.erb b/lib/data/templates/postgresql/postgresql.service.erb index 0bc60d9..71f10ce 100644 --- a/lib/data/templates/postgresql/postgresql.service.erb +++ b/lib/data/templates/postgresql/postgresql.service.erb @@ -5,8 +5,8 @@ After=network.target [Service] Type=notify -User=postgres -Group=services +User=postgresql +Group=servicesql ExecStart=<%= postgres_bin %> -D <%= database_dir %>/data ExecReload=/bin/kill -HUP $MAINPID diff --git a/lib/execute.rb b/lib/execute.rb index 782885a..550d21e 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -79,6 +79,8 @@ module Execute pdata = Make.context(name) bin_dir = pdata.get_prefix data_dir = "/data/#{pdata.name}/#{pdata.version.split(".").first}" + puts data_dir + exit -1 service_install_context = ServiceInstallContext.new( bin_dir, data_dir, name, pdata.version, @options.forced ) diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index 8feb8db..2efc8e3 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -9,7 +9,15 @@ module Setup module PostgreSQL extend Templates - + def self.write_as(user, path, content) + puts "wriiting by #{user} to #{path}" + # If executed as root we can just use File.write + # File.write(postgresql_conf_path, postgresql_conf_content) + IO.popen(["sudo", "-u", user.to_s, "tee", path], "w") do |io| + io.write(content) + end + end + # attr_accessor :te @@ -28,24 +36,31 @@ module Setup end def self.setup_systemd(context) + puts "DataDir: #{context.data_dir}" + pg_hba = render("pg_hba.conf") + pg_hba_path = "#{File.join(context.data_dir, "pg_hba.conf")}" + write_as(context.user_name, pg_hba_path, pg_hba) - end + postgresql_conf = render("postgresql.conf") + postgresql_conf_path = "#{File.join(context.data_dir, "postgresql.conf")}" + write_as(context.user_name, postgresql_conf_path, postgresql_conf) - def self.install(context) -# puts context.bin_dir - #make_config -# pg_hba = render("pg_hba.conf") -# -# # test = render("test", te: "This is a test string") -# posgresql_conf = render("postgresql") - service = render( + postgres_service = render( "postgresql.service", postgres_bin: File.join(context.bin_dir, "/bin/postgres"), version: context.version, database_dir: context.data_dir - ) - # puts service + ) + postgres_service_path = "/etc/systemd/system/postgresql.service" + write_as("root", postgres_service_path, postgres_service) + system("sudo systemctl daemon-reload") + system("sudo systemctl enable postgresql") + system("sudo systemctl start postgresql") + + end + + def self.install(context) user_context = Execute::UserInstallContext.new(context.user_name, :service) User.install(user_context) init_db(context) From dd1c82e80438e33b64136c8ca17b8e512d0a0c34 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Thu, 7 Aug 2025 21:05:37 +0200 Subject: [PATCH 22/52] more settings of systemd for postgress --- .../templates/postgresql/postgresql.conf.erb | 16 +++++++++------- .../templates/postgresql/postgresql.service.erb | 17 ++++++++++++----- lib/execute.rb | 3 --- lib/setup/postgresql.rb | 16 +++++++++++++--- recipes/postgresql/debian.yml | 2 +- 5 files changed, 35 insertions(+), 19 deletions(-) diff --git a/lib/data/templates/postgresql/postgresql.conf.erb b/lib/data/templates/postgresql/postgresql.conf.erb index c0617d1..b31cbfd 100644 --- a/lib/data/templates/postgresql/postgresql.conf.erb +++ b/lib/data/templates/postgresql/postgresql.conf.erb @@ -1,17 +1,19 @@ -data_directory = '/var/lib/postgresql/15/main' -hba_file = '/etc/postgresql/15/main/pg_hba.conf' -ident_file = '/etc/postgresql/15/main/pg_ident.conf' +#data_directory = '/var/lib/postgresql/15/main' +#hba_file = '/etc/postgresql/15/main/pg_hba.conf' +#ident_file = '/etc/postgresql/15/main/pg_ident.conf' #listen_addresses = 'localhost' listen_addresses = '*' port = 5432 max_connections = 100 -unix_socket_directories = '/var/run/postgresql' +unix_socket_directories = '<%= unix_socket %>' password_encryption = scram-sha-256 -ssl = on +## TODO Add support for ssl +#ssl = on + #ssl_ca_file = '' ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' #ssl_crl_file = '' @@ -36,7 +38,7 @@ log_line_prefix = '%m [%p] %q%u@%d ' log_timezone = 'Etc/UTC' -cluster_name = '15/main' +#cluster_name = '15/main' datestyle = 'iso, mdy' timezone = 'Etc/UTC' @@ -48,5 +50,5 @@ lc_time = 'C.UTF-8' default_text_search_config = 'pg_catalog.english' # include files ending in '.conf' from -include_dir = 'conf.d' +# include_dir = 'conf.d' diff --git a/lib/data/templates/postgresql/postgresql.service.erb b/lib/data/templates/postgresql/postgresql.service.erb index 71f10ce..bea3bc9 100644 --- a/lib/data/templates/postgresql/postgresql.service.erb +++ b/lib/data/templates/postgresql/postgresql.service.erb @@ -3,12 +3,17 @@ Description=PostgreSQL <%= version %> After=network.target [Service] -Type=notify +# Type=notify +Type=simple User=postgresql -Group=servicesql +Group=services + +ExecStart=<%= postgres_bin %> -D <%= database_dir %> + +StandardOutput=journal +StandardError=journal -ExecStart=<%= postgres_bin %> -D <%= database_dir %>/data ExecReload=/bin/kill -HUP $MAINPID KillMode=mixed TimeoutSec=300 @@ -16,8 +21,10 @@ Restart=on-failure NotifyAccess=all # Security -ProtectSystem=full -ProtectHome=true +#ProtectSystem=full +#ProtectHome=true +#ReadWritePaths=<%= database_dir %> + PrivateTmp=true NoNewPrivileges=true diff --git a/lib/execute.rb b/lib/execute.rb index 550d21e..ae52ea5 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -79,8 +79,6 @@ module Execute pdata = Make.context(name) bin_dir = pdata.get_prefix data_dir = "/data/#{pdata.name}/#{pdata.version.split(".").first}" - puts data_dir - exit -1 service_install_context = ServiceInstallContext.new( bin_dir, data_dir, name, pdata.version, @options.forced ) @@ -109,7 +107,6 @@ module Execute executor = get_install_executor(name) #executor.call(service_install_context) executor.call - exit -1 if block_given? context = OpenStruct.new diff --git a/lib/setup/postgresql.rb b/lib/setup/postgresql.rb index 2efc8e3..29007ac 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgresql.rb @@ -41,7 +41,15 @@ module Setup pg_hba_path = "#{File.join(context.data_dir, "pg_hba.conf")}" write_as(context.user_name, pg_hba_path, pg_hba) - postgresql_conf = render("postgresql.conf") + # TODO: move this to user module + uid = Etc.getpwnam(context.user_name.to_s).uid + socket_path = "/run/user/#{uid}" + system("sudo mkdir -p #{socket_path}") + system("sudo chown #{context.user_name}:services #{socket_path}") + system("sudo chmod 700 #{socket_path}") + + postgresql_conf = render("postgresql.conf", unix_socket: socket_path) + postgresql_conf_path = "#{File.join(context.data_dir, "postgresql.conf")}" write_as(context.user_name, postgresql_conf_path, postgresql_conf) @@ -53,11 +61,13 @@ module Setup ) postgres_service_path = "/etc/systemd/system/postgresql.service" write_as("root", postgres_service_path, postgres_service) - + system("sudo systemctl daemon-reexec") system("sudo systemctl daemon-reload") system("sudo systemctl enable postgresql") system("sudo systemctl start postgresql") - + + # debug service + # sudo systemctl daemon-reexec && sudo systemctl daemon-reload && sudo systemctl restart postgresql.service end def self.install(context) diff --git a/recipes/postgresql/debian.yml b/recipes/postgresql/debian.yml index fa81f49..d58d4c4 100644 --- a/recipes/postgresql/debian.yml +++ b/recipes/postgresql/debian.yml @@ -19,7 +19,7 @@ repository: version: 17.5 steps: - - ./configure --prefix=$PREFIX + - ./configure --prefix=$PREFIX --with-openssl --with-systemd - make -j$CPUS - $SUDO mkdir -p $PREFIX - $SUDO make install From 2054a68af0b76440438e6f888bd5bf3fcebf294e Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Fri, 8 Aug 2025 07:55:29 +0200 Subject: [PATCH 23/52] add command runner to nvim --- home/.config/nvim/init.lua | 3 +- home/.config/nvim/lua/command_runner.lua | 36 ++++++++++++++++++++++++ home/.config/nvim/lua/keymaps.lua | 5 ++++ recipes/nvim.yml | 5 ++-- recipes/ruby/debian.yml | 1 + 5 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 home/.config/nvim/lua/command_runner.lua diff --git a/home/.config/nvim/init.lua b/home/.config/nvim/init.lua index 1f37ecf..3b15ec1 100644 --- a/home/.config/nvim/init.lua +++ b/home/.config/nvim/init.lua @@ -1,4 +1,5 @@ +require("command_runner") require("plugins") require("options") require("keymaps") -require("lsp") \ No newline at end of file +require("lsp") diff --git a/home/.config/nvim/lua/command_runner.lua b/home/.config/nvim/lua/command_runner.lua new file mode 100644 index 0000000..012cb17 --- /dev/null +++ b/home/.config/nvim/lua/command_runner.lua @@ -0,0 +1,36 @@ +local M = {} + +function M.run_shell_command_under_cursor() + local cmd = vim.api.nvim_get_current_line() + + if cmd == "" then + print("No command on this line.") + return + end + + -- Run command and capture output + local output = vim.fn.systemlist(cmd) + + -- Save current window to return later + local current_win = vim.api.nvim_get_current_win() + + -- Move to the right window (assumes it's already split) + vim.cmd("wincmd l") + + -- Optional: clear buffer before writing + vim.api.nvim_buf_set_lines(0, 0, -1, false, {}) + + -- Set output in current buffer (right side) + vim.api.nvim_buf_set_lines(0, 0, -1, false, output) + + -- Optional: make it editable plain text + vim.bo.filetype = "text" + vim.bo.modifiable = true + vim.bo.readonly = false + + -- Return to the left window + vim.api.nvim_set_current_win(current_win) +end + +return M + diff --git a/home/.config/nvim/lua/keymaps.lua b/home/.config/nvim/lua/keymaps.lua index 48e81e4..4f469ea 100644 --- a/home/.config/nvim/lua/keymaps.lua +++ b/home/.config/nvim/lua/keymaps.lua @@ -5,6 +5,11 @@ map('n', 'fg', require('telescope.builtin').live_grep, { desc = "Live gr map('n', 'fb', require('telescope.builtin').buffers, { desc = "Buffers" }) map('n', 'fh', require('telescope.builtin').help_tags, { desc = "Help tags" }) +vim.keymap.set("n", "zr", function() + require("command_runner").run_shell_command_under_cursor() +end, { desc = "Run shell command under cursor and show output in right split" }) + + -- Project-specific ignored dirs for Telescope local function load_local_ignore() local cwd = vim.fn.getcwd() diff --git a/recipes/nvim.yml b/recipes/nvim.yml index 34d8313..d5492eb 100644 --- a/recipes/nvim.yml +++ b/recipes/nvim.yml @@ -5,8 +5,9 @@ packages: repository: url: https://github.com/neovim/neovim.git branch: release-0.11 + version: 0.11 steps: - make distclean - - make CMAKE_BUILD_TYPE=Release CMAKE_INSTALL_PREFIX=$HOME/.local - - make install + - make CMAKE_BUILD_TYPE=Release CMAKE_INSTALL_PREFIX=$PREFIX + - $SUDO make install diff --git a/recipes/ruby/debian.yml b/recipes/ruby/debian.yml index b715b8a..0079b0f 100644 --- a/recipes/ruby/debian.yml +++ b/recipes/ruby/debian.yml @@ -16,6 +16,7 @@ packages: repository: url: https://github.com/ruby/ruby.git branch: v3_4_5 + version: 3.4.5 steps: - ./autogen.sh From 2289379ad59b5f5d168acd06be5e8117d538b1dc Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Fri, 8 Aug 2025 10:17:01 +0200 Subject: [PATCH 24/52] add unix socket for postgres --- lib/data/templates/postgres/pg_hba.conf.erb | 3 +++ .../postgres.service.erb} | 10 ++++----- .../postgresql.conf.erb | 17 ++++++++------- lib/data/templates/postgresql/pg_hba.conf.erb | 2 -- lib/data/templates/postgresql/test.erb | 4 ---- lib/execute.rb | 4 ++-- lib/setup/{postgresql.rb => postgres.rb} | 21 +++++++++++-------- .../{postgresql => postgres}/debian-setup.yml | 0 recipes/{postgresql => postgres}/debian.yml | 0 9 files changed, 32 insertions(+), 29 deletions(-) create mode 100644 lib/data/templates/postgres/pg_hba.conf.erb rename lib/data/templates/{postgresql/postgresql.service.erb => postgres/postgres.service.erb} (82%) rename lib/data/templates/{postgresql => postgres}/postgresql.conf.erb (79%) delete mode 100644 lib/data/templates/postgresql/pg_hba.conf.erb delete mode 100644 lib/data/templates/postgresql/test.erb rename lib/setup/{postgresql.rb => postgres.rb} (79%) rename recipes/{postgresql => postgres}/debian-setup.yml (100%) rename recipes/{postgresql => postgres}/debian.yml (100%) diff --git a/lib/data/templates/postgres/pg_hba.conf.erb b/lib/data/templates/postgres/pg_hba.conf.erb new file mode 100644 index 0000000..6d1564e --- /dev/null +++ b/lib/data/templates/postgres/pg_hba.conf.erb @@ -0,0 +1,3 @@ +local all all peer +#hostssl all all 0.0.0.0/0 scram-sha-256 +host all all 127.0.0.1/32 trust diff --git a/lib/data/templates/postgresql/postgresql.service.erb b/lib/data/templates/postgres/postgres.service.erb similarity index 82% rename from lib/data/templates/postgresql/postgresql.service.erb rename to lib/data/templates/postgres/postgres.service.erb index bea3bc9..dd87eaf 100644 --- a/lib/data/templates/postgresql/postgresql.service.erb +++ b/lib/data/templates/postgres/postgres.service.erb @@ -3,10 +3,10 @@ Description=PostgreSQL <%= version %> After=network.target [Service] -# Type=notify -Type=simple +Type=notify +#Type=simple -User=postgresql +User=postgres Group=services ExecStart=<%= postgres_bin %> -D <%= database_dir %> @@ -23,9 +23,9 @@ NotifyAccess=all # Security #ProtectSystem=full #ProtectHome=true -#ReadWritePaths=<%= database_dir %> +ReadWritePaths=<%= database_dir %> -PrivateTmp=true +# PrivateTmp=true NoNewPrivileges=true # Resource Limits diff --git a/lib/data/templates/postgresql/postgresql.conf.erb b/lib/data/templates/postgres/postgresql.conf.erb similarity index 79% rename from lib/data/templates/postgresql/postgresql.conf.erb rename to lib/data/templates/postgres/postgresql.conf.erb index b31cbfd..2fb5871 100644 --- a/lib/data/templates/postgresql/postgresql.conf.erb +++ b/lib/data/templates/postgres/postgresql.conf.erb @@ -8,17 +8,20 @@ listen_addresses = '*' port = 5432 max_connections = 100 -unix_socket_directories = '<%= unix_socket %>' +#unix_socket_directories = '<%= unix_socket %>' +unix_socket_directories = '/tmp' password_encryption = scram-sha-256 -## TODO Add support for ssl +### TODO Add support for ssl #ssl = on +# +##ssl_ca_file = '' +#ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' +##ssl_crl_file = '' +##ssl_crl_dir = '' +#ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' + -#ssl_ca_file = '' -ssl_cert_file = '/etc/ssl/certs/ssl-cert-snakeoil.pem' -#ssl_crl_file = '' -#ssl_crl_dir = '' -ssl_key_file = '/etc/ssl/private/ssl-cert-snakeoil.key' #ssl_ciphers = 'HIGH:MEDIUM:+3DES:!aNULL' # allowed SSL ciphers #ssl_prefer_server_ciphers = on #ssl_ecdh_curve = 'prime256v1' diff --git a/lib/data/templates/postgresql/pg_hba.conf.erb b/lib/data/templates/postgresql/pg_hba.conf.erb deleted file mode 100644 index 1afb456..0000000 --- a/lib/data/templates/postgresql/pg_hba.conf.erb +++ /dev/null @@ -1,2 +0,0 @@ -local all all peer -hostssl all all 0.0.0.0/0 scram-sha-256 diff --git a/lib/data/templates/postgresql/test.erb b/lib/data/templates/postgresql/test.erb deleted file mode 100644 index b0a5afa..0000000 --- a/lib/data/templates/postgresql/test.erb +++ /dev/null @@ -1,4 +0,0 @@ - - - -<%= te %> diff --git a/lib/execute.rb b/lib/execute.rb index ae52ea5..2bd9ed9 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -84,8 +84,8 @@ module Execute ) case name - when :postgresql - require 'setup/postgresql' + when :postgres + require 'setup/postgres' -> { Setup::PostgreSQL.install(service_install_context) } # ->(context) { # Setup::PostgreSQL.install(context) diff --git a/lib/setup/postgresql.rb b/lib/setup/postgres.rb similarity index 79% rename from lib/setup/postgresql.rb rename to lib/setup/postgres.rb index 29007ac..fc08516 100644 --- a/lib/setup/postgresql.rb +++ b/lib/setup/postgres.rb @@ -32,7 +32,7 @@ module Setup end system("sudo mkdir -p #{context.data_dir}") system("sudo chown #{context.user_name}:services #{context.data_dir}") - system("sudo -u postgresql #{File.join(context.bin_dir, "/bin/initdb")} -D #{context.data_dir}") + system("sudo -u postgres #{File.join(context.bin_dir, "/bin/initdb")} -D #{context.data_dir} --username=postgres") end def self.setup_systemd(context) @@ -41,12 +41,15 @@ module Setup pg_hba_path = "#{File.join(context.data_dir, "pg_hba.conf")}" write_as(context.user_name, pg_hba_path, pg_hba) - # TODO: move this to user module - uid = Etc.getpwnam(context.user_name.to_s).uid - socket_path = "/run/user/#{uid}" + ## TODO: move this to user module + #uid = Etc.getpwnam(context.user_name.to_s).uid + #socket_path = "/run/user/#{uid}" + socket_path = "/run/user/#{context.user_name}" + #socket_path = "/tmp" + system("sudo mkdir -p #{socket_path}") system("sudo chown #{context.user_name}:services #{socket_path}") - system("sudo chmod 700 #{socket_path}") + system("sudo chmod 711 #{socket_path}") postgresql_conf = render("postgresql.conf", unix_socket: socket_path) @@ -54,17 +57,17 @@ module Setup write_as(context.user_name, postgresql_conf_path, postgresql_conf) postgres_service = render( - "postgresql.service", + "postgres.service", postgres_bin: File.join(context.bin_dir, "/bin/postgres"), version: context.version, database_dir: context.data_dir ) - postgres_service_path = "/etc/systemd/system/postgresql.service" + postgres_service_path = "/etc/systemd/system/postgres.service" write_as("root", postgres_service_path, postgres_service) system("sudo systemctl daemon-reexec") system("sudo systemctl daemon-reload") - system("sudo systemctl enable postgresql") - system("sudo systemctl start postgresql") + system("sudo systemctl enable postgres") + system("sudo systemctl start postgres") # debug service # sudo systemctl daemon-reexec && sudo systemctl daemon-reload && sudo systemctl restart postgresql.service diff --git a/recipes/postgresql/debian-setup.yml b/recipes/postgres/debian-setup.yml similarity index 100% rename from recipes/postgresql/debian-setup.yml rename to recipes/postgres/debian-setup.yml diff --git a/recipes/postgresql/debian.yml b/recipes/postgres/debian.yml similarity index 100% rename from recipes/postgresql/debian.yml rename to recipes/postgres/debian.yml From 9c3189544ad6686472d44a2c496e8826fce22a0b Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Fri, 8 Aug 2025 10:36:58 +0200 Subject: [PATCH 25/52] Postgres: Listen on two sockets --- lib/data/templates/postgres/postgresql.conf.erb | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/data/templates/postgres/postgresql.conf.erb b/lib/data/templates/postgres/postgresql.conf.erb index 2fb5871..f257924 100644 --- a/lib/data/templates/postgres/postgresql.conf.erb +++ b/lib/data/templates/postgres/postgresql.conf.erb @@ -8,8 +8,8 @@ listen_addresses = '*' port = 5432 max_connections = 100 -#unix_socket_directories = '<%= unix_socket %>' -unix_socket_directories = '/tmp' +unix_socket_directories = '/tmp, <%= unix_socket %>' +# unix_socket_directories = '/tmp' password_encryption = scram-sha-256 ### TODO Add support for ssl From 75c3228c5eeded1be2fb71913ba5b72b12d61199 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sat, 9 Aug 2025 18:10:34 +0200 Subject: [PATCH 26/52] add script for creating VM image --- bin/certbot | 7 +++ bin/make-winux | 122 +++++++++++++++++++++++++++++++++++++ bin/use | 1 + lib/certbot.rb | 13 ++++ lib/execute.rb | 7 +++ recipes/certbot/debian.yml | 21 +++++++ recipes/gcc.yml | 4 +- recipes/nginx/debian.yml | 23 +++++++ 8 files changed, 196 insertions(+), 2 deletions(-) create mode 100755 bin/certbot create mode 100755 bin/make-winux create mode 100644 bin/use create mode 100644 lib/certbot.rb create mode 100644 recipes/certbot/debian.yml create mode 100644 recipes/nginx/debian.yml diff --git a/bin/certbot b/bin/certbot new file mode 100755 index 0000000..d759648 --- /dev/null +++ b/bin/certbot @@ -0,0 +1,7 @@ +#!/usr/bin/env zsh + +export PYTHONPATH="/pkg/certbot/4.2.0/local/lib/python3.11/dist-packages" +export PATH="/pkg/certbot/4.2.0/local/bin:$PATH" + +/pkg/certbot/4.2.0/local/bin/certbot + diff --git a/bin/make-winux b/bin/make-winux new file mode 100755 index 0000000..777489c --- /dev/null +++ b/bin/make-winux @@ -0,0 +1,122 @@ +#!/usr/bin/env zsh + +export LINUX_DISK_IMG=/home/artur/winux/VM/linux/linux.img +export WINUX_DISK_IMG=./winux.img + +# Mount the image of oryginal +# + +# Mounting the image +function wxumount { + echo "unmounting :$1" + mkdir -p $1 + if mountpoint -q $1; then + echo "Unmounting $1..." + umount /mnt/winux + if [ $? -eq 0 ]; then + echo "Successfully unmounted $1." + else + echo "Failed to unmount $1." + fi + else + echo "$1 is not mounted." + fi +} + +wxumount /mnt/winux +wxumount /mnt/linux + +# (re)create an empty sparsed file +rm -rf $WINUX_DISK_IMG +dd if=/dev/zero of=$WINUX_DISK_IMG bs=1 count=0 seek=20G + +# Check size +# real size: du -h winux.img +# appeared as: ls -lh winux.img + + +# Copy boot + +IMG="/home/artur/winux/VM/linux/linux.img" + +SECTOR_SIZE=$(LC_ALL=C sfdisk -d "$LINUX_DISK_IMG" | awk 'match($0,/sector-size:\s*([0-9]+)/,m){print m[1]; exit}') +START=$(LC_ALL=C sfdisk -d "$LINUX_DISK_IMG" | awk 'match($0,/start=\s*([0-9]+)/,m){print m[1]; exit}') + +echo "Sector size: $SECTOR_SIZE" +echo "Start sector: $START" + + +dd if="$LINUX_DISK_IMG" of="$WINUX_DISK_IMG" bs="$SECTOR_SIZE" count="$START" conv=notrunc + +# Formatting the disk +# No altering the first 1MB +echo '2048,,83,*' | sudo sfdisk --no-reread $WINUX_DISK_IMG + +# List all loops: sudo losetup -a +WINUX_LOOP_DEV=$(sudo losetup --find --show $WINUX_DISK_IMG) +# Scan also for partitions: sudo losetup --find --show --partscan winux.img + +echo "Loop device: $WINUX_LOOP_DEV" + + + +export WINUX_LOOP_DEV="$(losetup --find --show -P $WINUX_LOOP_DEV)" + +mkfs.ext2 ${WINUX_LOOP_DEV}p1 + +mount ${WINUX_LOOP_DEV}p1 /mnt/winux + +## The image is formatted and ready to copy files over + + +export LINUX_LOOP_DEV="$(losetup --find --show --partscan $LINUX_DISK_IMG)" +mount ${LINUX_LOOP_DEV}p1 /mnt/linux + + +# copying files + +cd /mnt/linux +tar --numeric-owner --xattrs --acls -cpf - \ + --exclude='lost+found' \ + --exclude='var/log/*' \ + --exclude='var/tmp/*' \ + --exclude='tmp/*' \ + --exclude='var/cache/*' \ + --exclude='swapfile' \ + --exclude='dev/*' \ + --exclude='proc/*' \ + --exclude='sys/*' \ + --exclude='run/*' \ + . | tar --numeric-owner --xattrs --acls -xpf - -C /mnt/winux + + + +#### TESTTING CODE +losetup -j $WINUX_DISK_IMG + +# Clean up after building + +wxumount /mnt/winux +wxumount /mnt/linux + +for dev in $(losetup -j $WINUX_DISK_IMG | cut -d: -f1); do + losetup -d "$dev" +done + +for dev in $(losetup -j $LINUX_DISK_IMG | cut -d: -f1); do + losetup -d "$dev" +done + + +# Create an archive +# + +echo "making the archive" +zstd -T0 --ultra -22 winux.img -o winux.img.zst + +# decompression with +# unzstd --sparse winux.img.zst +# or +# zstd -d --sparse winux.img.zst -o winux.img + + diff --git a/bin/use b/bin/use new file mode 100644 index 0000000..c179399 --- /dev/null +++ b/bin/use @@ -0,0 +1 @@ +use python 3.11 diff --git a/lib/certbot.rb b/lib/certbot.rb new file mode 100644 index 0000000..32c3b39 --- /dev/null +++ b/lib/certbot.rb @@ -0,0 +1,13 @@ + +module Certbot + + def self.create_certificate(domain, wildcard = false) + command = "/home/artur/.dat/bin/certbot" + if wildcard + system "sudo #{command} certonly --manual --preferred-challenges=dns -d \"*.#{domain}\" -d \"#{domain}\"" + else + system "sudo #{command} --nginx -d #{domain}" + end + end + +end diff --git a/lib/execute.rb b/lib/execute.rb index 2bd9ed9..3418959 100644 --- a/lib/execute.rb +++ b/lib/execute.rb @@ -119,6 +119,13 @@ module Execute end end + def domain(name) + require 'certbot' + wildcard = name.strip.start_with?("*.") + domain = name.strip.delete_prefix("*.") + Certbot.create_certificate(domain, wildcard) + end + def user(*users) require 'user' users.each do |name| diff --git a/recipes/certbot/debian.yml b/recipes/certbot/debian.yml new file mode 100644 index 0000000..904e3fe --- /dev/null +++ b/recipes/certbot/debian.yml @@ -0,0 +1,21 @@ +environment: + PYTHONPATH: /home/artur/test + +dependencies: + - python + +packages: + - libffi-dev + - libssl-dev + +repository: + url: https://github.com/certbot/certbot + branch: v4.2.0 + version: 4.2.0 + +steps: + - $SUDO pip3 install --prefix=$PREFIX certbot + - $SUDO pip3 install --prefix=$PREFIX certbot-nginx + + # can be executed like: pip install --prefix=$PREFIX certbot certbot-nginx + # to see installed plugins: `certbot plugins` diff --git a/recipes/gcc.yml b/recipes/gcc.yml index 0cd1029..f668a08 100644 --- a/recipes/gcc.yml +++ b/recipes/gcc.yml @@ -14,6 +14,6 @@ repository: steps: - ./contrib/download_prerequisites - - ./configure --prefix=$HOME/.local --enable-languages=c,c++ --disable-multilib + - ./configure --prefix=$PREFIX --enable-languages=c,c++ --disable-multilib - make -j$(nproc) - - make install + - $SUDO make install diff --git a/recipes/nginx/debian.yml b/recipes/nginx/debian.yml new file mode 100644 index 0000000..ed9772c --- /dev/null +++ b/recipes/nginx/debian.yml @@ -0,0 +1,23 @@ + +packages: + - build-essential + - libpcre3 + - libpcre3-dev + - zlib1g + - zlib1g-dev + - libssl-dev + +repository: + url: https://github.com/nginx/nginx.git + branch: release-1.29.0 + version: 1.29.0 + +build: + - ./configure \| + --prefix=$PREFIX \| + --with-http_ssl_module \| + --with-http_gzip_static_module + --with-stream + --with-http_v2_module + - make + - $SUDO make install From 045d586fb636095e4a6114257b0d567b3c5eb1de Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 07:30:01 +0200 Subject: [PATCH 27/52] add compression --- bin/make-winux | 25 +++++++++++++++++++------ 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/bin/make-winux b/bin/make-winux index 777489c..073592f 100755 --- a/bin/make-winux +++ b/bin/make-winux @@ -111,12 +111,25 @@ done # Create an archive # -echo "making the archive" -zstd -T0 --ultra -22 winux.img -o winux.img.zst +# echo "making the archive" +# zstd -T0 --ultra -22 winux.img -o winux.img.zst -# decompression with -# unzstd --sparse winux.img.zst -# or -# zstd -d --sparse winux.img.zst -o winux.img +# # decompression with +# # unzstd --sparse winux.img.zst +# # or +# # zstd -d --sparse winux.img.zst -o winux.img +############# Another option, seems to be the best option +tar --sparse -I 'zstd -T0 --ultra -22 --long=31' \ + -cvf winux.img.tar.zst winux.img +# extract +# brew install gnu-tar +# tar -I zstd -xvf winux.img.tar.zst +# on macOS /opt/homebrew/opt/gnu-tar/libexec/gnubin/tar -I 'zstd --long=31' -xvf winux.img.tar.zst winux.img + +# compressing with XZ. Supprisely it seems to have worst compresion rate than zstd + +# tar --sparse -I 'xz -T0 -9e' -cvf winux.img.tar.xz winux.img +## Extracting +# tar -I xz -xvf winux.img.tar.xz From 8085ed43a2d757aaff93e38de7ff659611dcab81 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 07:55:21 +0200 Subject: [PATCH 28/52] make compressing faster and try to fix booting the image --- bin/make-winux | 31 +++++++++++++++++++++---------- 1 file changed, 21 insertions(+), 10 deletions(-) diff --git a/bin/make-winux b/bin/make-winux index 073592f..d579ad8 100755 --- a/bin/make-winux +++ b/bin/make-winux @@ -70,6 +70,9 @@ mount ${WINUX_LOOP_DEV}p1 /mnt/winux export LINUX_LOOP_DEV="$(losetup --find --show --partscan $LINUX_DISK_IMG)" + +tune2fs ${WINUX_LOOP_DEV}p1 -U 1be261e2-a12d-4468-aa45-cbd7e68636eb + mount ${LINUX_LOOP_DEV}p1 /mnt/linux @@ -111,25 +114,33 @@ done # Create an archive # -# echo "making the archive" -# zstd -T0 --ultra -22 winux.img -o winux.img.zst +echo "making the archive" -# # decompression with -# # unzstd --sparse winux.img.zst -# # or -# # zstd -d --sparse winux.img.zst -o winux.img +# Best compressing rato, but slow and memory hungry (I LIKE IT) +#tar --sparse -I 'zstd -T0 --ultra -22 --long=31' \ +# -cvf winux.img.tar.zst winux.img +## SO I PICKED NOW FOR SPEED +tar --sparse -I 'zstd -T0 -9' -cvf winux.img.tar.zst winux.img -############# Another option, seems to be the best option -tar --sparse -I 'zstd -T0 --ultra -22 --long=31' \ - -cvf winux.img.tar.zst winux.img # extract # brew install gnu-tar # tar -I zstd -xvf winux.img.tar.zst -# on macOS /opt/homebrew/opt/gnu-tar/libexec/gnubin/tar -I 'zstd --long=31' -xvf winux.img.tar.zst winux.img +# on macOS gtar or /opt/homebrew/opt/gnu-tar/libexec/gnubin/tar -I 'zstd --long=31' -xvf winux.img.tar.zst winux.img + + +# FINISHED +#======================================================= # compressing with XZ. Supprisely it seems to have worst compresion rate than zstd # tar --sparse -I 'xz -T0 -9e' -cvf winux.img.tar.xz winux.img ## Extracting # tar -I xz -xvf winux.img.tar.xz +# +# zstd -T0 --ultra -22 winux.img -o winux.img.zst + +# # decompression with +# # unzstd --sparse winux.img.zst +# # or +# # zstd -d --sparse winux.img.zst -o winux.img From 82c27b7221723f87d338f9c644715bb6ef2b055b Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 08:05:57 +0200 Subject: [PATCH 29/52] Fix bug with umounting --- bin/make-winux | 15 ++++----------- 1 file changed, 4 insertions(+), 11 deletions(-) diff --git a/bin/make-winux b/bin/make-winux index d579ad8..3a8343d 100755 --- a/bin/make-winux +++ b/bin/make-winux @@ -12,7 +12,7 @@ function wxumount { mkdir -p $1 if mountpoint -q $1; then echo "Unmounting $1..." - umount /mnt/winux + umount $1 if [ $? -eq 0 ]; then echo "Successfully unmounted $1." else @@ -64,18 +64,15 @@ export WINUX_LOOP_DEV="$(losetup --find --show -P $WINUX_LOOP_DEV)" mkfs.ext2 ${WINUX_LOOP_DEV}p1 +tune2fs ${WINUX_LOOP_DEV}p1 -U 1be261e2-a12d-4468-aa45-cbd7e68636eb + mount ${WINUX_LOOP_DEV}p1 /mnt/winux ## The image is formatted and ready to copy files over - export LINUX_LOOP_DEV="$(losetup --find --show --partscan $LINUX_DISK_IMG)" - -tune2fs ${WINUX_LOOP_DEV}p1 -U 1be261e2-a12d-4468-aa45-cbd7e68636eb - mount ${LINUX_LOOP_DEV}p1 /mnt/linux - # copying files cd /mnt/linux @@ -95,7 +92,7 @@ tar --numeric-owner --xattrs --acls -cpf - \ #### TESTTING CODE -losetup -j $WINUX_DISK_IMG +# losetup -j $WINUX_DISK_IMG # Clean up after building @@ -110,10 +107,6 @@ for dev in $(losetup -j $LINUX_DISK_IMG | cut -d: -f1); do losetup -d "$dev" done - -# Create an archive -# - echo "making the archive" # Best compressing rato, but slow and memory hungry (I LIKE IT) From 6ad86cf1d402f9ed177ce76334d1194097599a0e Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 09:06:58 +0200 Subject: [PATCH 30/52] Creating VM image is working --- bin/make-winux | 39 ++++++++++++++++++++++++++------------- 1 file changed, 26 insertions(+), 13 deletions(-) diff --git a/bin/make-winux b/bin/make-winux index 3a8343d..27fda43 100755 --- a/bin/make-winux +++ b/bin/make-winux @@ -1,6 +1,8 @@ #!/usr/bin/env zsh -export LINUX_DISK_IMG=/home/artur/winux/VM/linux/linux.img +#export LINUX_DISK_IMG=/home/artur/winux/VM/linux/linux.img +export LINUX_DISK_IMG=/home/artur/winux/VM/deb-orig/debian.img + export WINUX_DISK_IMG=./winux.img # Mount the image of oryginal @@ -9,18 +11,26 @@ export WINUX_DISK_IMG=./winux.img # Mounting the image function wxumount { echo "unmounting :$1" - mkdir -p $1 - if mountpoint -q $1; then - echo "Unmounting $1..." - umount $1 - if [ $? -eq 0 ]; then - echo "Successfully unmounted $1." - else - echo "Failed to unmount $1." - fi + while lsof +D $1 >/dev/null 2>&1 || fuser $1 >/dev/null 2>&1 + do + fuser -a $1 + echo "Waiting for $1 to be free..." + sleep 1 + done + + + mkdir -p $1 + if mountpoint -q $1; then + echo "Unmounting $1..." + umount $1 + if [ $? -eq 0 ]; then + echo "Successfully unmounted $1." else - echo "$1 is not mounted." + echo "Failed to unmount $1." fi + else + echo "$1 is not mounted." + fi } wxumount /mnt/winux @@ -74,7 +84,7 @@ export LINUX_LOOP_DEV="$(losetup --find --show --partscan $LINUX_DISK_IMG)" mount ${LINUX_LOOP_DEV}p1 /mnt/linux # copying files - +SAVED_PWD=`pwd` cd /mnt/linux tar --numeric-owner --xattrs --acls -cpf - \ --exclude='lost+found' \ @@ -89,13 +99,16 @@ tar --numeric-owner --xattrs --acls -cpf - \ --exclude='run/*' \ . | tar --numeric-owner --xattrs --acls -xpf - -C /mnt/winux - +cd "$SAVED_PWD" #### TESTTING CODE # losetup -j $WINUX_DISK_IMG # Clean up after building +sync +sleep 1 + wxumount /mnt/winux wxumount /mnt/linux From 5737f5ee50e38a5352a6dba3bb06d0bb03799a70 Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 11:06:53 +0200 Subject: [PATCH 31/52] save --- bin/set-zsh | 1 + bin/use | 3 +++ recipes/nginx/debian.yml | 32 +++++++++++++++++++++++++------- recipes/nginx/nginx.service.erb | 23 +++++++++++++++++++++++ 4 files changed, 52 insertions(+), 7 deletions(-) create mode 100644 bin/set-zsh create mode 100644 recipes/nginx/nginx.service.erb diff --git a/bin/set-zsh b/bin/set-zsh new file mode 100644 index 0000000..3143ac3 --- /dev/null +++ b/bin/set-zsh @@ -0,0 +1 @@ +chsh -s $(which zsh) diff --git a/bin/use b/bin/use index c179399..d1b250d 100644 --- a/bin/use +++ b/bin/use @@ -1 +1,4 @@ use python 3.11 + + +.zshrc-using < setting the environment for the user/serice diff --git a/recipes/nginx/debian.yml b/recipes/nginx/debian.yml index ed9772c..3d19034 100644 --- a/recipes/nginx/debian.yml +++ b/recipes/nginx/debian.yml @@ -12,12 +12,30 @@ repository: branch: release-1.29.0 version: 1.29.0 -build: - - ./configure \| - --prefix=$PREFIX \| - --with-http_ssl_module \| - --with-http_gzip_static_module - --with-stream - --with-http_v2_module +# TODO: finish the configuration: last message of 6898551e-70a0-8331-8617-bcfc9bcf6af8 +steps: + - echo "starting" + - | + ./auto/configure \ + --prefix=$PREFIX \ + --with-http_ssl_module \ + --with-http_gzip_static_module \ + --with-stream \ + --with-http_v2_module \ + --with-pcre-jit \ + --with-threads \ + --with-file-aio \ + --with-http_ssl_module \ + --with-http_v2_module \ + --with-http_gzip_static_module \ + --with-stream=dynamic \ + --with-stream_ssl_module \ + --sbin-path=$PREFIX/nginx/sbin/nginx \ + --conf-path=$PREFIX/nginx/conf/nginx.conf \ + --pid-path=$PREFIX/nginx/run/nginx.pid \ + --lock-path=$PREFIX/nginx/run/nginx.lock \ + --error-log-path=$PREFIX/nginx/logs/error.log \ + --http-log-path=$PREFIX/nginx/logs/access.log + - make - $SUDO make install diff --git a/recipes/nginx/nginx.service.erb b/recipes/nginx/nginx.service.erb new file mode 100644 index 0000000..7b9ebaa --- /dev/null +++ b/recipes/nginx/nginx.service.erb @@ -0,0 +1,23 @@ +sudo tee /etc/systemd/system/nginx-custom.service >/dev/null <<'EOF' +[Unit] +Description=nginx (custom prefix) +After=network.target + +[Service] +Type=forking +PIDFile=/pkg/nginx/run/nginx.pid +ExecStart=/pkg/nginx/sbin/nginx +ExecReload=/pkg/nginx/sbin/nginx -s reload +ExecStop=/pkg/nginx/sbin/nginx -s quit +TimeoutStopSec=5 +#User=nginx +#Group=nginx +LimitNOFILE=65535 + +[Install] +WantedBy=multi-user.target +EOF + +# sudo systemctl daemon-reload +# sudo systemctl enable --now nginx-custom + From 8789922ba68cb1320dddb403a379b13705a9503a Mon Sep 17 00:00:00 2001 From: Artur Gurgul1 Date: Sun, 10 Aug 2025 20:12:10 +0200 Subject: [PATCH 32/52] save --- bin/vm | 27 --------------------------- 1 file changed, 27 deletions(-) diff --git a/bin/vm b/bin/vm index ed29799..e69de29 100755 --- a/bin/vm +++ b/bin/vm @@ -1,27 +0,0 @@ - -# mount -t 9p -o trans=virtio,version=9p2000.L share /home/user - -# this add to brake and inspect /dev if /dev/sda1 not found -# -append "root=/dev/sda1 console=ttyS0 rd.break" - -function run { - qemu-system-x86_64 -append "root=/dev/sda1 console=ttyS0 rd.break" \ - -kernel "vmlinuz-linux" \ - -initrd "initramfs-linux.img" \ - -m 2048 \ - -smp $(sysctl -n hw.logicalcpu) \ - -cpu qemu64 \ - -virtfs local,path=.,security_model=none,mount_tag=share \ - -drive id=root-disk,if=none,format=raw,file=linux.img \ - -device ide-hd,bus=ide.0,drive=root-disk \ - -drive id=data-disk,if=none,format=qcow2,file=dat.qcow2 \ - -device ide-hd,bus=ide.1,drive=data-disk \ - -nographic -} - - -function reset { - qemu-img create -f qcow2 dat.qcow2 32G -} - -$1 From c08110ae3895a5ae6c8388310a42620a930febc7 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 11 Aug 2025 10:20:07 +0200 Subject: [PATCH 33/52] Add script for seting up media center --- bin/admin/single-user | 65 ++++++++++++++++++++++++++++++++ bin/vm | 34 +++++++++++++++++ bin/zshrc/init | 4 ++ lib/data/resources/iso-images.rb | 21 +++++++++++ lib/downloader.rb | 21 +++++++++++ lib/virtual-machine.rb | 16 ++++++++ 6 files changed, 161 insertions(+) create mode 100644 bin/admin/single-user create mode 100644 lib/data/resources/iso-images.rb create mode 100644 lib/downloader.rb create mode 100644 lib/virtual-machine.rb diff --git a/bin/admin/single-user b/bin/admin/single-user new file mode 100644 index 0000000..4b647ff --- /dev/null +++ b/bin/admin/single-user @@ -0,0 +1,65 @@ +#!/bin/bash +## This script make autologin. If user has phisical access +## to the device we can trust him, otherwiese we require password +## for remote access + + +# Parameters +USERNAME="user" + +# Sanity check +if [ "$(id -u)" -ne 0 ]; then + echo "Please run this script as root." + exit 1 +fi + +if ! id "$USERNAME" &>/dev/null; then + echo "User '$USERNAME' does not exist." + exit 1 +fi + +echo "Setting up passwordless sudo for physical access for user: $USERNAME" +echo + +# 1. Configure sudoers: allow passwordless sudo +echo "Configuring sudoers for $USERNAME..." +echo "Defaults:$USERNAME !authenticate" >> /etc/sudoers.d/00-$USERNAME-nopasswd +chmod 440 /etc/sudoers.d/00-$USERNAME-nopasswd + +# 2. Modify PAM for sudo to allow password only on non-physical ttys +echo "Configuring PAM for sudo tty check..." +PAM_FILE="/etc/pam.d/sudo" +BACKUP_FILE="/etc/pam.d/sudo.bak" + +if ! grep -q "pam_succeed_if.so tty" "$PAM_FILE"; then + echo "Creating backup of $PAM_FILE to $BACKUP_FILE" + cp "$PAM_FILE" "$BACKUP_FILE" + + sed -i '1iauth [success=1 default=ignore] pam_succeed_if.so tty =~ /dev/tty[0-9]*' "$PAM_FILE" + echo "PAM modified to restrict passwordless sudo to physical TTYs." +else + echo "PAM sudo already appears configured." +fi + +# 3. Enable autologin on tty1 +echo "Configuring systemd autologin on tty1 for $USERNAME..." +mkdir -p /etc/systemd/system/getty@tty1.service.d +AUTOLOGIN_CONF="/etc/systemd/system/getty@tty1.service.d/override.conf" + +cat > "$AUTOLOGIN_CONF" <" --name "" + +options = OpenStruct.new +subcommand = ARGV.shift&.to_sym +options.parameter = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil + +OptionParser.new do |opt| + opt.on('--arch ARCH', 'Architecture arm64 or x86_64') do |arch| + options.arch = arch + end + + opt.on('--name NAME', 'Virtaul Machine name') do |arch| + options.arch = arch + end + +end.parse! + + +case subcommand +when :create + puts "Creating image...." + +when :setup + options[:distro] = subcommand + VirtualMachine.setup(options) +else + puts "Error not found #{options.type}" +end \ No newline at end of file diff --git a/bin/zshrc/init b/bin/zshrc/init index e6acd5c..79c3372 100644 --- a/bin/zshrc/init +++ b/bin/zshrc/init @@ -19,6 +19,10 @@ function cdd { cd $DAT_ROOT } +function cdp { + cd $PASSWORD_STORE_DIR +} + . $DAT_ROOT/bin/zshrc/prompt . $DAT_ROOT/bin/zshrc/utils diff --git a/lib/data/resources/iso-images.rb b/lib/data/resources/iso-images.rb new file mode 100644 index 0000000..4f6ef76 --- /dev/null +++ b/lib/data/resources/iso-images.rb @@ -0,0 +1,21 @@ + + +module VirtualMachine + ISO_URLS = { + debian: { + arm64: { + install: "https://cdimage.debian.org/debian-cd/current/arm64/iso-cd/debian-13.0.0-arm64-netinst.iso" + }, + x86_64: { + install: "https://cdimage.debian.org/debian-cd/current/amd64/iso-cd/debian-13.0.0-amd64-netinst.iso", + live: "https://cdimage.debian.org/debian-cd/current-live/amd64/iso-hybrid/debian-live-13.0.0-amd64-standard.iso" + } + }, + archlinux: { + x86_64: { + live: "https://geo.mirror.pkgbuild.com/iso/2025.08.01/archlinux-x86_64.iso" + } + } + } + +end \ No newline at end of file diff --git a/lib/downloader.rb b/lib/downloader.rb new file mode 100644 index 0000000..6fa1e67 --- /dev/null +++ b/lib/downloader.rb @@ -0,0 +1,21 @@ +require 'uri' +require 'addressable/uri' + + +module Downloader + # use_cache => save in the home and return path to it + # forced => download even if the file exists in the cache, saves to chace if use_cache == true + def self.get(url, use_cache = true, forced = false) + puts "downloading..." + puts url + + + uri = Addressable::URI.parse(url) + path = File.join("#{ENV["HOME"]}/.cache/dat/downloads/", uri.domain, uri.path) + + #uri = URI.parse(url) + #file_name = File.basename(uri.path) + puts "Download path: #{path}" + yield path + end +end \ No newline at end of file diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb new file mode 100644 index 0000000..b27bbcb --- /dev/null +++ b/lib/virtual-machine.rb @@ -0,0 +1,16 @@ +require 'downloader' +require_relative 'data/resources/iso-images' + +module VirtualMachine + def self.distro(name, arch, type = :install) + ISO_URLS[:debian][:arm64][:install] + end + + def self.setup(options) + distro = options[:distro] + arch = :arm64 + type = :install + url = distro(name, arch, type) + Downloader.get(url) + end +end \ No newline at end of file From 3b9e0a1f33b2e0b102e2fd5fde9c2bcc2143c957 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 11 Aug 2025 10:30:52 +0200 Subject: [PATCH 34/52] Fixed version by ChatGPT --- bin/admin/single-user | 73 ++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 29 deletions(-) mode change 100644 => 100755 bin/admin/single-user diff --git a/bin/admin/single-user b/bin/admin/single-user old mode 100644 new mode 100755 index 4b647ff..85018fc --- a/bin/admin/single-user +++ b/bin/admin/single-user @@ -4,62 +4,77 @@ ## for remote access -# Parameters +#!/bin/bash +set -euo pipefail + +# Settings USERNAME="user" -# Sanity check if [ "$(id -u)" -ne 0 ]; then - echo "Please run this script as root." - exit 1 + echo "Please run this script as root." >&2 + exit 1 fi -if ! id "$USERNAME" &>/dev/null; then - echo "User '$USERNAME' does not exist." - exit 1 -fi +id "$USERNAME" &>/dev/null || { echo "User '$USERNAME' does not exist." >&2; exit 1; } -echo "Setting up passwordless sudo for physical access for user: $USERNAME" +echo "Configuring passwordless sudo ONLY on local TTYs for $USERNAME" echo -# 1. Configure sudoers: allow passwordless sudo -echo "Configuring sudoers for $USERNAME..." -echo "Defaults:$USERNAME !authenticate" >> /etc/sudoers.d/00-$USERNAME-nopasswd -chmod 440 /etc/sudoers.d/00-$USERNAME-nopasswd +SUDOERS_FILE="/etc/sudoers.d/90-$USERNAME-sudo" +if [ ! -f "$SUDOERS_FILE" ]; then + echo "$USERNAME ALL=(ALL:ALL) ALL" > "$SUDOERS_FILE" + chmod 0440 "$SUDOERS_FILE" + visudo -cf "$SUDOERS_FILE" >/dev/null || { echo "sudoers validation failed"; exit 1; } +fi -# 2. Modify PAM for sudo to allow password only on non-physical ttys -echo "Configuring PAM for sudo tty check..." +# PAM: allow passwordless sudo on physical ttys, require password otherwise. PAM_FILE="/etc/pam.d/sudo" BACKUP_FILE="/etc/pam.d/sudo.bak" -if ! grep -q "pam_succeed_if.so tty" "$PAM_FILE"; then - echo "Creating backup of $PAM_FILE to $BACKUP_FILE" - cp "$PAM_FILE" "$BACKUP_FILE" +if ! grep -q "BEGIN local TTY passwordless" "$PAM_FILE"; then + echo "Backing up $PAM_FILE to $BACKUP_FILE" + cp "$PAM_FILE" "$BACKUP_FILE" - sed -i '1iauth [success=1 default=ignore] pam_succeed_if.so tty =~ /dev/tty[0-9]*' "$PAM_FILE" - echo "PAM modified to restrict passwordless sudo to physical TTYs." + # Insert at the top of the file: + # - If TTY matches /dev/ttyN (real VT), immediately permit auth (no password). + # - Otherwise fall through to the normal rules (which will ask for a password). + # + # Notes: + # - pam_succeed_if.so is in the libpam-modules package on Debian/Ubuntu (ensure installed). + # - The 'success=done' short-circuits the auth stack on real TTYs. + tmp="$(mktemp)" + cat > "$tmp" <<'PAMHEAD' +# --- BEGIN local TTY passwordless --- +# Passwordless sudo on real virtual consoles only: +# /dev/tty1, /dev/tty2, ... (NOT /dev/pts/* used by SSH) +auth [success=done default=ignore] pam_succeed_if.so tty =~ ^/dev/tty[0-9]+$ +# --- END local TTY passwordless --- +PAMHEAD + cat "$PAM_FILE" >> "$tmp" + mv "$tmp" "$PAM_FILE" + echo "PAM updated. On SSH (/dev/pts/*) a password will be required." else - echo "PAM sudo already appears configured." + echo "PAM sudo already has local TTY rule." fi -# 3. Enable autologin on tty1 +# 3) Autologin on tty1 (physical console) echo "Configuring systemd autologin on tty1 for $USERNAME..." mkdir -p /etc/systemd/system/getty@tty1.service.d AUTOLOGIN_CONF="/etc/systemd/system/getty@tty1.service.d/override.conf" +AGETTY_BIN="$(command -v agetty || true)" +: "${AGETTY_BIN:=/sbin/agetty}" cat > "$AUTOLOGIN_CONF" < Date: Mon, 11 Aug 2025 14:18:59 +0200 Subject: [PATCH 35/52] creating raw images for QEMU --- bin/admin/install-gui | 7 +++++ bin/vm | 8 ++--- install | 6 ++-- lib/certbot.rb | 2 -- lib/downloader.rb | 26 ++++++++++++----- lib/local-user/single-user.rb | 10 +++++++ lib/system.rb | 8 +++++ lib/system/architecture.rb | 55 +++++++++++++++++++++++++++++++++++ lib/system/debian.rb | 5 ++++ lib/system/macos.rb | 5 ++++ lib/system/utils.rb | 6 ++++ lib/user.rb | 4 +++ lib/virtual-machine.rb | 39 +++++++++++++++++++++++-- 13 files changed, 164 insertions(+), 17 deletions(-) create mode 100755 bin/admin/install-gui create mode 100644 lib/local-user/single-user.rb create mode 100644 lib/system/architecture.rb create mode 100644 lib/system/utils.rb diff --git a/bin/admin/install-gui b/bin/admin/install-gui new file mode 100755 index 0000000..3043f9d --- /dev/null +++ b/bin/admin/install-gui @@ -0,0 +1,7 @@ + +## missing in the install script +apt install zsh curl + +apt install sway libgtk-3-0 libasound2 + +wget https://github.com/zen-browser/desktop/releases/download/1.14.11b/zen.linux-x86_64.tar.xz \ No newline at end of file diff --git a/bin/vm b/bin/vm index eb6eee3..0675205 100755 --- a/bin/vm +++ b/bin/vm @@ -8,15 +8,15 @@ require 'virtual-machine' options = OpenStruct.new subcommand = ARGV.shift&.to_sym -options.parameter = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil +parameter = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil OptionParser.new do |opt| opt.on('--arch ARCH', 'Architecture arm64 or x86_64') do |arch| options.arch = arch end - opt.on('--name NAME', 'Virtaul Machine name') do |arch| - options.arch = arch + opt.on('--name NAME', 'Virtaul Machine name') do |name| + options.name = name end end.parse! @@ -27,7 +27,7 @@ when :create puts "Creating image...." when :setup - options[:distro] = subcommand + options[:distro] = parameter VirtualMachine.setup(options) else puts "Error not found #{options.type}" diff --git a/install b/install index 1b8b6c8..7dcfed1 100755 --- a/install +++ b/install @@ -79,7 +79,7 @@ export DAT_ROOT=$(echo "$DAT_ROOT" | envsubst) debian_install_packages() { # List of required packages - local packages=("git" "ruby") + local packages=("git" "ruby" "zsh") local to_install=() # Determine if we need to use sudo @@ -105,11 +105,13 @@ debian_install_packages() { else echo "All required packages are already installed." fi + + chsh -s $(which zsh) } macos_install_packages() { # List of required packages - local packages=("git" "ruby") + local packages=("git" "ruby" "zsh") local to_install=() # Check for Homebrew diff --git a/lib/certbot.rb b/lib/certbot.rb index 32c3b39..836495b 100644 --- a/lib/certbot.rb +++ b/lib/certbot.rb @@ -1,6 +1,5 @@ module Certbot - def self.create_certificate(domain, wildcard = false) command = "/home/artur/.dat/bin/certbot" if wildcard @@ -9,5 +8,4 @@ module Certbot system "sudo #{command} --nginx -d #{domain}" end end - end diff --git a/lib/downloader.rb b/lib/downloader.rb index 6fa1e67..462be6e 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -1,21 +1,33 @@ require 'uri' require 'addressable/uri' - +require 'user' +require 'fileutils' +require 'open-uri' module Downloader # use_cache => save in the home and return path to it # forced => download even if the file exists in the cache, saves to chace if use_cache == true def self.get(url, use_cache = true, forced = false) - puts "downloading..." - puts url - - uri = Addressable::URI.parse(url) - path = File.join("#{ENV["HOME"]}/.cache/dat/downloads/", uri.domain, uri.path) + path = File.join(User.cache_path, "downloads", uri.domain, uri.path) #uri = URI.parse(url) #file_name = File.basename(uri.path) - puts "Download path: #{path}" + + + # Ensure the directory exists + dir = File.dirname(path) + FileUtils.mkdir_p(dir) unless Dir.exist?(dir) + + unless File.exist?(path) + puts "File does not exist, downloading..." + URI.open(uri) do |input| + File.open(path, 'wb') do |output| + IO.copy_stream(input, output) + end + end + end + yield path end end \ No newline at end of file diff --git a/lib/local-user/single-user.rb b/lib/local-user/single-user.rb new file mode 100644 index 0000000..d055093 --- /dev/null +++ b/lib/local-user/single-user.rb @@ -0,0 +1,10 @@ + + +module User + + # Make user to be trusted when he use + # the computer locally + def make_local_trusted(user) + File.mkdir_p("/etc/systemd/system/getty@tty1.service.d") + end +end \ No newline at end of file diff --git a/lib/system.rb b/lib/system.rb index 14aee45..28d7f38 100644 --- a/lib/system.rb +++ b/lib/system.rb @@ -1,5 +1,7 @@ +require_relative('system/architecture') module System + def self.detect_os case RUBY_PLATFORM when /darwin/ @@ -29,7 +31,13 @@ module System raise "Operating system not supported" end + ARCH = normalize_architecture_string(arch) + def self.os_info puts os_name end + + def self.arch_to_symbol(arch) + normalize_architecture_string(arch) + end end diff --git a/lib/system/architecture.rb b/lib/system/architecture.rb new file mode 100644 index 0000000..657ac23 --- /dev/null +++ b/lib/system/architecture.rb @@ -0,0 +1,55 @@ +module System + def self.normalize_architecture_string(s) + str = s.to_s.strip.downcase + exact = { + "x86_64" => :x86_64, "amd64" => :x86_64, + "i386" => :x86_32, "i486" => :x86_32, "i586" => :x86_32, "i686" => :x86_32, + + "arm64" => :arm64, "aarch64" => :arm64, + "armhf" => :armv7, "armel" => :armv5, "armv7l" => :armv7, "armv6l" => :armv6, "armv5tel" => :armv5, + + "riscv64" => :riscv64, + + "ppc64el" => :ppc64le, "ppc64le" => :ppc64le, + "ppc64" => :ppc64, "powerpc64" => :ppc64, + "ppc" => :ppc32, "powerpc" => :ppc32, + + "s390x" => :s390x, + + "mips64el" => :mips64el, "mips64" => :mips64, + "mipsel" => :mipsel, "mips" => :mips, + + "loong64" => :loongarch64, "loongarch64" => :loongarch64, + + "sparc64" => :sparc64, + + "alpha" => :alpha, + "hppa" => :hppa, + "ia64" => :ia64, + } + return exact[str] if exact.key?(str) + + return :x86_64 if str.include?("x64") || str.include?("amd64") + return :x86_32 if str =~ /\Ai[3-6]86\z/ || str.include?("x86") + return :arm64 if str.include?("aarch64") || str.include?("arm64") + return :armv7 if str.include?("armv7") + return :armv6 if str.include?("armv6") + return :armv5 if str.include?("armv5") + return :riscv64 if str.include?("riscv64") + return :ppc64le if str.include?("ppc64el") || str.include?("ppc64le") + return :ppc64 if str.include?("ppc64") + return :ppc32 if str.include?("ppc") || str.include?("powerpc") + return :s390x if str.include?("s390x") + return :mips64el if str.include?("mips64el") + return :mips64 if str.include?("mips64") + return :mipsel if str.include?("mipsel") + return :mips if str.start_with?("mips") + return :loongarch64 if str.include?("loong") || str.include?("loongarch") + return :sparc64 if str.include?("sparc64") + return :alpha if str.include?("alpha") + return :hppa if str.include?("hppa") + return :ia64 if str.include?("ia64") + + :unknown + end +end \ No newline at end of file diff --git a/lib/system/debian.rb b/lib/system/debian.rb index 6504a2f..c7803df 100644 --- a/lib/system/debian.rb +++ b/lib/system/debian.rb @@ -1,4 +1,5 @@ require 'open3' +require_relative "utils" module DebianSystem def os_name @@ -10,6 +11,10 @@ module DebianSystem stdout.strip.to_i end + def arch + sh("dpkg --print-architecture") + end + def install(packages) missing_packages = packages.reject { |pkg| package_installed?(pkg) } diff --git a/lib/system/macos.rb b/lib/system/macos.rb index b2a6a22..cc3c9ff 100644 --- a/lib/system/macos.rb +++ b/lib/system/macos.rb @@ -1,10 +1,15 @@ require 'open3' +require_relative "utils" module MacOSSystem def os_name "macOS" end + def arch + sh("sysctl -n hw.machine") + end + def cpus stdout, stderr, status = Open3.capture3("sysctl -n hw.ncpu") stdout.strip.to_i diff --git a/lib/system/utils.rb b/lib/system/utils.rb new file mode 100644 index 0000000..70d98c8 --- /dev/null +++ b/lib/system/utils.rb @@ -0,0 +1,6 @@ +def sh(cmd) + out = `#{cmd} 2>/dev/null`.to_s.strip + out.empty? ? "" : out + rescue + "" +end \ No newline at end of file diff --git a/lib/user.rb b/lib/user.rb index b084396..724436b 100644 --- a/lib/user.rb +++ b/lib/user.rb @@ -34,4 +34,8 @@ module User end end + + def self.cache_path + "#{ENV["HOME"]}/.cache/dat/" + end end diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index b27bbcb..44dae19 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -1,4 +1,5 @@ require 'downloader' +require 'system' require_relative 'data/resources/iso-images' module VirtualMachine @@ -7,10 +8,44 @@ module VirtualMachine end def self.setup(options) + if options[:name] == nil + options[:name] = options[:distro] + end + + if options[:arch] == nil + options[:arch] = System::ARCH + else + options[:arch] = System.arch_to_symbol(options[:arch]) + end + + puts options + distro = options[:distro] - arch = :arm64 type = :install + arch = options[:arch] url = distro(name, arch, type) - Downloader.get(url) + + Downloader.get(url) do |path| + disk_img_path = File.join(User.cache_path, "vm", distro.to_s, arch.to_s, options[:name], "root.img") + create_disk_image(disk_img_path, 500) + + puts path + puts disk_img_path + end + end + + # size in MB + def self.create_disk_image(path, size) + size_in_bytes = 1024 * 1024 * size + + # Ensure the directory exists + dir = File.dirname(path) + FileUtils.mkdir_p(dir) unless Dir.exist?(dir) + + File.open(path, "wb") do |f| + f.truncate(size_in_bytes) + # f.seek(size_in_bytes - 1) + # f.write("\0") + end end end \ No newline at end of file From 78d7233ff3190e1a11ce1c2a05cd4b4096367f00 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Mon, 11 Aug 2025 17:36:55 +0200 Subject: [PATCH 36/52] Working version of QEMU configuration --- lib/downloader.rb | 2 +- lib/virtual-machine.rb | 12 +++- lib/vm/qemu.rb | 126 +++++++++++++++++++++++++++++++++++++++++ 3 files changed, 138 insertions(+), 2 deletions(-) create mode 100644 lib/vm/qemu.rb diff --git a/lib/downloader.rb b/lib/downloader.rb index 462be6e..3ca652a 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -23,7 +23,7 @@ module Downloader puts "File does not exist, downloading..." URI.open(uri) do |input| File.open(path, 'wb') do |output| - IO.copy_stream(input, output) + IO.copy_stream(input, output) end end end diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 44dae19..2f5476d 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -1,6 +1,7 @@ require 'downloader' require 'system' require_relative 'data/resources/iso-images' +require 'vm/qemu' module VirtualMachine def self.distro(name, arch, type = :install) @@ -27,14 +28,23 @@ module VirtualMachine Downloader.get(url) do |path| disk_img_path = File.join(User.cache_path, "vm", distro.to_s, arch.to_s, options[:name], "root.img") - create_disk_image(disk_img_path, 500) + create_disk_image(disk_img_path, 5000) puts path puts disk_img_path + + Qemu.launch( + arch, + disk_img_path, + cpus: [1, System.cpus - 2].max, + cdrom: path, + detach: false + ) end end # size in MB + # lsof /Users/artur/.cache/dat/vm/debian/arm64/debian/root.img def self.create_disk_image(path, size) size_in_bytes = 1024 * 1024 * size diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb new file mode 100644 index 0000000..905e56a --- /dev/null +++ b/lib/vm/qemu.rb @@ -0,0 +1,126 @@ +require "fileutils" +require "ostruct" + +module Qemu + def self.qemu_bin_for(arch) + { + x86_64: "qemu-system-x86_64", + x86_32: "qemu-system-i386", + arm64: "qemu-system-aarch64", + armv7: "qemu-system-arm", + armv6: "qemu-system-arm", + armv5: "qemu-system-arm", + riscv64: "qemu-system-riscv64", + ppc64le: "qemu-system-ppc64", + ppc64: "qemu-system-ppc64", + ppc32: "qemu-system-ppc", + s390x: "qemu-system-s390x", + mips64el: "qemu-system-mips64el", + mips64: "qemu-system-mips64", + mipsel: "qemu-system-mipsel", + mips: "qemu-system-mips", + loongarch64: "qemu-system-loongarch64", + sparc64: "qemu-system-sparc64", + alpha: "qemu-system-alpha", + hppa: "qemu-system-hppa", + ia64: "qemu-system-ia64", + }.fetch(arch) { raise "Unsupported arch: #{arch.inspect}" } + end + + def self.accel_args + host = RbConfig::CONFIG["host_os"] + if host =~ /linux/i && File.exist?("/dev/kvm") + ["-accel", "kvm"] + elsif host =~ /darwin/i + # hvf exists on Apple Silicon + Intel macOS; QEMU falls back if not available + ["-accel", "hvf"] + elsif host =~ /freebsd/i + # if QEMU was built with it; otherwise it will ignore + ["-accel", "bhyve"] + else + ["-accel", "tcg"] + end + end + + def self.machine_args_for(arch) + case arch + when :x86_64, :x86_32 + [] + when :arm64 + # -machine type=virt + # -cpu cortex-a72 + ["-machine", "virt", "-cpu", "max"] + when :armv7, :armv6, :armv5 + ["-machine", "virt", "-cpu", "cortex-a15"] + when :riscv64 + ["-machine", "virt"] + when :loongarch64 + ["-machine", "virt"] + else + [] + end + end + + #def self.launch(arch, disk_path, cdrom = nil, detach = true) + def self.launch(arch, disk_path, **options) + defaults = { + arch: System::ARCH, + cdrom: nil, + detach: true, + ram: 2048, + cpus: 1 + } + opts = defaults.merge(options) + + puts opts + + qemu = qemu_bin_for(arch) + args = [] + + if System::OS == :macos && arch == :arm64 + # args += ["-bios", "/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] + args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] + + #args += ["-device", "virtio-gpu-device"] + args += ["-display", "cocoa"] + args += ["-device", "qemu-xhci,id=xhci"] + args += ["-device", "usb-kbd"] + args += ["-device", "usb-tablet"] + + args += ["-device", "virtio-keyboard-device"] + args += ["-device", "virtio-mouse-device"] + args += ["-device", "virtio-gpu"] + + + # copy to /Users/artur/.cache/dat/vm/debian/arm64/debian/vars.fd + # /opt/homebrew/share/qemu/edk2-aarch64-vars.fd + # args += ["-drive", "if=pflash,format=raw,unit=1,file=/Users/artur/.cache/dat/vm/debian/arm64/debian/vars.fd"] + end + + args += accel_args + args += machine_args_for(arch) + args += ["-m", opts[:ram].to_s, "-smp", opts[:cpus].to_s] + args += ["-name", "FirstVM", "-boot", "order=d"] # boot from CD first + args += ["-drive", "file=#{disk_path},if=virtio,cache=writeback,format=raw"] + if opts[:cdrom] != nil + args += ["-cdrom", opts[:cdrom]] + end + args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT + # optional: uncomment to run headless with VNC on :5901 + # args += ["-display", "none", "-vnc", "127.0.0.1:1"] + + cmd = [qemu, *args] + + puts "Launching: #{cmd.join(' ')}" + pid = Process.spawn(*cmd, pgroup: true, out: $stdout, err: $stderr) + + if opts[:detach] + Process.detach(pid) + puts "QEMU pid=#{pid}" + else + Process.wait(pid) + status = $? + puts "Exit status: #{status.exitstatus}" + end + end +end \ No newline at end of file From ca5d4c4b72dc4839389c2e3d42eb986d0c3378a2 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 12 Aug 2025 07:35:18 +0200 Subject: [PATCH 37/52] Example of using SSL: CA => CSR => Sign --- bin/ssl | 93 ++++++++++++++++++++++++++++++++++++++++++ bin/vm | 3 +- lib/virtual-machine.rb | 2 +- 3 files changed, 96 insertions(+), 2 deletions(-) create mode 100755 bin/ssl diff --git a/bin/ssl b/bin/ssl new file mode 100755 index 0000000..60ce748 --- /dev/null +++ b/bin/ssl @@ -0,0 +1,93 @@ +#!/usr/bin/env ruby + +require "fileutils" +require "open3" + +def cmd(*cmd) + stdout, stderr, status = Open3.capture3(*cmd) + unless status.success? + warn "Command failed: #{cmd.join(' ')}" + warn stderr + exit status.exitstatus || 1 + end + stdout +end + +subcommand = ARGV.shift&.to_sym +arg1 = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil +arg2 = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil + +$domains = ["mediacenter.lan"] +$ips = ["192.168.0.94"] + + +### Setting up for client +$cn = $domains.first +name = $cn.gsub("*", "wildcard") + +dir = "certs" +FileUtils.mkdir_p(dir) +$key = File.join(dir, "#{name}.key.pem") +$csr = File.join(dir, "#{name}.csr.pem") +$crt = File.join(dir, "#{name}.crt.pem") +$ext = File.join(dir, "#{name}.ext") +############################ + +### Setting up for CA + +$ca_name = "My Lab" + +############################# + +def create_CA + cmd "openssl", "genrsa", "-out", "rootCA.key.pem", "4096" + cmd "chmod", "600", "rootCA.key.pem" + cmd "openssl", "req", "-x509", "-new", "-nodes", "-key", + "rootCA.key.pem", "-sha256", "-days", "3650", + "-out", "rootCA.crt.pem", + "-subj", "/C=XX/ST=Lab/L=Local/O=#{$ca_name}/CN=#{$ca_name} Root CA" +end + +def create_CSR + cmd "openssl", "genrsa", "-out", $key, "2048" + cmd "openssl", "req", "-new", "-key", $key, "-out", $csr, "-subj", "/CN=#{$cn}/O=#{$ca_name}" +end + +def create_extfile + ext_lines = [] + ext_lines << "basicConstraints=CA:FALSE" + ext_lines << "keyUsage=digitalSignature,keyEncipherment" + ext_lines << "extendedKeyUsage=serverAuth" + ext_lines << "subjectAltName=@alt_names" + ext_lines << "[alt_names]" + + $domains.each_with_index do |d, i| + ext_lines << "DNS.#{i + 1}=#{d}" + end + + $ips.each_with_index do |ip, j| + ext_lines << "IP.#{j + 1}=#{ip}" + end + + File.write($ext, ext_lines.join("\n") + "\n") +end + +def sign_with_CA + cmd "openssl", "x509", "-req", "-in", $csr, + "-CA", "rootCA.crt.pem", "-CAkey", "rootCA.key.pem", + "-CAcreateserial", "-out", $crt, "-days", "397", "-sha256", + "-extfile", $ext + +end + +case subcommand +when :ca + create_CA +when :csr + create_CSR +when :casign + create_extfile + sign_with_CA +else + puts "no command handler" +end \ No newline at end of file diff --git a/bin/vm b/bin/vm index 0675205..983fb1f 100755 --- a/bin/vm +++ b/bin/vm @@ -18,7 +18,8 @@ OptionParser.new do |opt| opt.on('--name NAME', 'Virtaul Machine name') do |name| options.name = name end - + # new from copy + # fetch from the server end.parse! diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 2f5476d..3875625 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -28,7 +28,7 @@ module VirtualMachine Downloader.get(url) do |path| disk_img_path = File.join(User.cache_path, "vm", distro.to_s, arch.to_s, options[:name], "root.img") - create_disk_image(disk_img_path, 5000) + create_disk_image(disk_img_path, 15000) puts path puts disk_img_path From 8bbf7d31a5d85ee4fe4a38863383fb0b5f16300f Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 13 Aug 2025 07:06:53 +0200 Subject: [PATCH 38/52] add ca service --- bin/admin/certbot-cert | 4 ++++ bin/admin/install-idf | 21 +++++++++++++++++++++ bin/admin/install-idf-enable | 2 ++ bin/services/ca/Gemfile | 6 ++++++ bin/services/ca/Gemfile.lock | 26 ++++++++++++++++++++++++++ bin/services/ca/app.rb | 12 ++++++++++++ bin/services/ca/config.ru | 2 ++ bin/services/ca/make | 0 bin/services/ca/readme.md | 9 +++++++++ bin/ssl | 8 +++++--- 10 files changed, 87 insertions(+), 3 deletions(-) create mode 100644 bin/admin/certbot-cert create mode 100644 bin/admin/install-idf create mode 100644 bin/admin/install-idf-enable create mode 100644 bin/services/ca/Gemfile create mode 100644 bin/services/ca/Gemfile.lock create mode 100644 bin/services/ca/app.rb create mode 100644 bin/services/ca/config.ru create mode 100755 bin/services/ca/make create mode 100644 bin/services/ca/readme.md diff --git a/bin/admin/certbot-cert b/bin/admin/certbot-cert new file mode 100644 index 0000000..e06ff98 --- /dev/null +++ b/bin/admin/certbot-cert @@ -0,0 +1,4 @@ +#/usr/bin/env bash + +certbot certonly --manual --preferred-challenges=dns -d "*.gurgul.org" -d "gurgul.org" + diff --git a/bin/admin/install-idf b/bin/admin/install-idf new file mode 100644 index 0000000..06d9939 --- /dev/null +++ b/bin/admin/install-idf @@ -0,0 +1,21 @@ +if [ -z "$SUDO_USER" ]; then + echo "This script must be run within sudo." + exit 1 +fi + +# from says ~/esp +ESP=/pkg/esp + +apt-get install git wget flex bison gperf python3 python3-pip python3-venv cmake ninja-build ccache libffi-dev libssl-dev dfu-util libusb-1.0-0 + +sudo -u "$SUDO_USER" bash <= 3.0" +gem "roda" +gem "puma" +gem "rackup" \ No newline at end of file diff --git a/bin/services/ca/Gemfile.lock b/bin/services/ca/Gemfile.lock new file mode 100644 index 0000000..d81fe89 --- /dev/null +++ b/bin/services/ca/Gemfile.lock @@ -0,0 +1,26 @@ +GEM + remote: https://rubygems.org/ + specs: + nio4r (2.7.4) + puma (6.6.1) + nio4r (~> 2.0) + rack (3.2.0) + rackup (2.2.1) + rack (>= 3) + roda (3.95.0) + rack + +PLATFORMS + arm64-darwin-24 + ruby + +DEPENDENCIES + puma + rackup + roda + +RUBY VERSION + ruby 3.4.5p51 + +BUNDLED WITH + 2.6.9 diff --git a/bin/services/ca/app.rb b/bin/services/ca/app.rb new file mode 100644 index 0000000..46a803c --- /dev/null +++ b/bin/services/ca/app.rb @@ -0,0 +1,12 @@ +require "roda" + +class App < Roda + # automatically sets application/json and encodes Ruby objects + plugin :json + + route do |r| + r.get "hello" do + { message: "world" } + end + end +end \ No newline at end of file diff --git a/bin/services/ca/config.ru b/bin/services/ca/config.ru new file mode 100644 index 0000000..d829dc7 --- /dev/null +++ b/bin/services/ca/config.ru @@ -0,0 +1,2 @@ +require_relative "app" +run App.freeze.app \ No newline at end of file diff --git a/bin/services/ca/make b/bin/services/ca/make new file mode 100755 index 0000000..e69de29 diff --git a/bin/services/ca/readme.md b/bin/services/ca/readme.md new file mode 100644 index 0000000..211b003 --- /dev/null +++ b/bin/services/ca/readme.md @@ -0,0 +1,9 @@ + +bundle exec rackup -p 9292 -o 0.0.0.0 + + + +RACK_ENV=production bundle exec rackup -p 8080 -o 0.0.0.0 + + +curl http://0.0.0.0:9292/hello --verbose \ No newline at end of file diff --git a/bin/ssl b/bin/ssl index 60ce748..973d60a 100755 --- a/bin/ssl +++ b/bin/ssl @@ -17,9 +17,11 @@ subcommand = ARGV.shift&.to_sym arg1 = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil arg2 = ARGV[0] && ARGV[0] !~ /^-/ ? ARGV.shift : nil -$domains = ["mediacenter.lan"] -$ips = ["192.168.0.94"] - +# $domains = ["mediacenter.lan"] +# $ips = ["192.168.0.94"] +$domains = ["testing.self"] +$ips = ["127.0.0.2"] +$ips = [] ### Setting up for client $cn = $domains.first From 2840b6e221a6febe361333ae94bb8b5b80877033 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 13 Aug 2025 10:21:46 +0200 Subject: [PATCH 39/52] Add option for archive and restore VMs --- bin/vm | 39 ++++++++++++++++++- lib/virtual-machine.rb | 87 ++++++++++++++++++++++++++++++++++-------- lib/vm/archive.rb | 37 ++++++++++++++++++ 3 files changed, 146 insertions(+), 17 deletions(-) create mode 100644 lib/vm/archive.rb diff --git a/bin/vm b/bin/vm index 983fb1f..969248b 100755 --- a/bin/vm +++ b/bin/vm @@ -4,6 +4,10 @@ require 'optparse' require 'ostruct' require 'virtual-machine' +# Paramteters +# -env: DAT_VM_DATA=~/.cache/vm + + # vm setup debian --arch "" --name "" options = OpenStruct.new @@ -18,6 +22,8 @@ OptionParser.new do |opt| opt.on('--name NAME', 'Virtaul Machine name') do |name| options.name = name end + # --port (ssh port) + # --attached (do not leave the shell process) # new from copy # fetch from the server end.parse! @@ -25,11 +31,40 @@ end.parse! case subcommand when :create - puts "Creating image...." - + puts "vm create debian-copy --name debian" + puts "Creating image base on existing one...." +when :run + # same as setup but fetches image not + puts "Run the image or fetch and run...." + options[:distro] = parameter + VirtualMachine.run(options) + when :setup options[:distro] = parameter VirtualMachine.setup(options) +when :archive + # whole image or files that are not present in the archived + # --type files (zst all files by files) + # by default + #before start if should create index of files that are present on + # the vm, so later we can copy new files or changed + + options[:distro] = parameter + VirtualMachine.archive(options) + +when :appy + puts "appy chnages from VM to the host os" +when :restore + options[:distro] = parameter + VirtualMachine.restore(options) + #puts "download image from remote or if local copy exists then the local copy" +when :daemon + puts "setup a pprocess so i can attach to it" + # paramaters exatly as run/setup +when :status + puts "Print all images and status stopped/daemon(port)/running(port/)" +when :attach + puts "if not paramteres attaches to last used" else puts "Error not found #{options.type}" end \ No newline at end of file diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 3875625..ef696b9 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -2,13 +2,46 @@ require 'downloader' require 'system' require_relative 'data/resources/iso-images' require 'vm/qemu' +require 'vm/archive' module VirtualMachine def self.distro(name, arch, type = :install) - ISO_URLS[:debian][:arm64][:install] + ISO_URLS[name][arch][type] end - def self.setup(options) + def self.vm_root_path + ENV["DAT_VM_DATA"] || File.join(User.cache_path, "vm") + end + + def self.vm_dir(options) + File.join(vm_root_path, "image", "#{options[:distro].to_s}-#{options[:arch].to_s}", options[:name]) + end + + def self.archive_dir(options) + File.join(vm_root_path, "archive", "#{options[:distro].to_s}-#{options[:arch].to_s}", options[:name]) + end + + def self.create_archive_path(options) + File.join(archive_dir(options), "archive.tar.zst") + end + + def self.get_recent_archive_path(options) + File.join(archive_dir(options), "archive.tar.zst") + end + + # https://artur.gurgul.pro/vm/ + # https://artur.gurgul.pro/vm/debian-arm64/debian/archive.tar.zst + + def self.vm_archive_url(options) + return nil unless ENV["DAT_VM_REPO_URL"] + + end + + def self.root_img_path(options) + File.join(vm_dir(options), "root.img") + end + + def self.fill_defaults(options) if options[:name] == nil options[:name] = options[:distro] end @@ -18,27 +51,51 @@ module VirtualMachine else options[:arch] = System.arch_to_symbol(options[:arch]) end + end - puts options + def self.archive(options) + fill_defaults(options) + Archive.create(vm_dir(options), out: create_archive_path(options)) + end - distro = options[:distro] - type = :install - arch = options[:arch] - url = distro(name, arch, type) + def self.restore(options) + fill_defaults(options) + Archive.restore(get_recent_archive_path(options), vm_dir(options)) + end + + def self.run(options) + fill_defaults(options) + + disk_img_path = root_img_path(options) + # TODO: + # - if image path not exists, check the cache + # - if cache do not exists try to download + puts "Starting image: #{disk_img_path}" + + Qemu.launch( + options[:arch], + disk_img_path, + cpus: [1, System.cpus - 2].max, + detach: true + ) + end + + def self.setup(options) + fill_defaults(options) - Downloader.get(url) do |path| - disk_img_path = File.join(User.cache_path, "vm", distro.to_s, arch.to_s, options[:name], "root.img") - create_disk_image(disk_img_path, 15000) - - puts path - puts disk_img_path + url = distro(options[:name], options[:arch], :install) + disk_img_path = root_img_path(options) + Downloader.get(url) do |path| + + create_disk_image(disk_img_path, 64000) + Qemu.launch( - arch, + options[:arch], disk_img_path, cpus: [1, System.cpus - 2].max, cdrom: path, - detach: false + detach: true ) end end diff --git a/lib/vm/archive.rb b/lib/vm/archive.rb new file mode 100644 index 0000000..8554f15 --- /dev/null +++ b/lib/vm/archive.rb @@ -0,0 +1,37 @@ +require "fileutils" + +module Archive + def self.cmd(*cmd) + puts "cmd: #{cmd.join(" ")}" + # return + stdout, stderr, status = Open3.capture3(*cmd) + unless status.success? + warn "Command failed: #{cmd.join(' ')}" + warn stderr + exit status.exitstatus || 1 + end + stdout + end + + # it preserved sparsiveness + # gtar -S --sparse-version=2 -I 'zstd -T0 -19' -cpf test.tar.zst /Volumes/Projs/VM/VM-data/image/debian-arm64/debian/ + # tar --zstd -xpf test.tar.zst -C ./r + + def self.create(path, **options) + out = options[:out] || File.join(Dir.pwd, "archive.tar.zst") + + FileUtils.mkdir_p(File.dirname(out)) + cmd "gtar", "-S", "--sparse-version=2", "-I", + "zstd -T0 -19", "-cpf", out, "-C" , path, "." + end + + def self.restore(file, path, **options) + puts file + puts path + puts options + + FileUtils.mkdir_p(path) + cmd "gtar", "-S", "--sparse-version=2", "-I", "zstd", "-xpf", + file, "-C", path + end +end \ No newline at end of file From f8d67e621be4b169bd29e8c901aacb3b697d19e8 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Thu, 14 Aug 2025 08:03:10 +0200 Subject: [PATCH 40/52] Download image if not exists on the system --- bin/vm | 7 +++++-- lib/downloader.rb | 1 + lib/virtual-machine.rb | 20 +++++++++++++++++--- 3 files changed, 23 insertions(+), 5 deletions(-) diff --git a/bin/vm b/bin/vm index 969248b..657bb10 100755 --- a/bin/vm +++ b/bin/vm @@ -52,8 +52,8 @@ when :archive options[:distro] = parameter VirtualMachine.archive(options) -when :appy - puts "appy chnages from VM to the host os" +when :apply + puts "apply chnages from VM to the host os" when :restore options[:distro] = parameter VirtualMachine.restore(options) @@ -61,6 +61,9 @@ when :restore when :daemon puts "setup a pprocess so i can attach to it" # paramaters exatly as run/setup +when :execute + puts "vm execute debian --file example.sh --attach (block terminal till done otherwiese notifi when done)" + puts "if name not provided then use the last accessed" when :status puts "Print all images and status stopped/daemon(port)/running(port/)" when :attach diff --git a/lib/downloader.rb b/lib/downloader.rb index 3ca652a..1095ab8 100644 --- a/lib/downloader.rb +++ b/lib/downloader.rb @@ -5,6 +5,7 @@ require 'fileutils' require 'open-uri' module Downloader + # brew install gnu-tar # use_cache => save in the home and return path to it # forced => download even if the file exists in the cache, saves to chace if use_cache == true def self.get(url, use_cache = true, forced = false) diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index ef696b9..27abeb7 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -9,16 +9,20 @@ module VirtualMachine ISO_URLS[name][arch][type] end + def self.image_id_dir(options) + File.join("#{options[:distro].to_s}-#{options[:arch].to_s}", options[:name]) + end + def self.vm_root_path ENV["DAT_VM_DATA"] || File.join(User.cache_path, "vm") end def self.vm_dir(options) - File.join(vm_root_path, "image", "#{options[:distro].to_s}-#{options[:arch].to_s}", options[:name]) + File.join(vm_root_path, "image", image_id_dir(options)) end def self.archive_dir(options) - File.join(vm_root_path, "archive", "#{options[:distro].to_s}-#{options[:arch].to_s}", options[:name]) + File.join(vm_root_path, "archive", image_id_dir(options)) end def self.create_archive_path(options) @@ -34,7 +38,10 @@ module VirtualMachine def self.vm_archive_url(options) return nil unless ENV["DAT_VM_REPO_URL"] - + image_id = image_id_dir(options) + uri = Addressable::URI.parse(ENV["DAT_VM_REPO_URL"]) + uri.path = File.join(uri.path, image_id, "archive.tar.zst") + return uri.to_s end def self.root_img_path(options) @@ -67,6 +74,13 @@ module VirtualMachine fill_defaults(options) disk_img_path = root_img_path(options) + unless File.exist?(disk_img_path) + url = vm_archive_url(options) + Downloader.get(url, use_cache: false) do |path| + Archive.restore(path, vm_dir(options)) + end + #raise "file do not exists" + end # TODO: # - if image path not exists, check the cache # - if cache do not exists try to download From bf52a151ff07d710cd9770050600232c8942d1a9 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Thu, 14 Aug 2025 14:55:49 +0200 Subject: [PATCH 41/52] search for password --- bin/password | 5 +++++ 1 file changed, 5 insertions(+) create mode 100755 bin/password diff --git a/bin/password b/bin/password new file mode 100755 index 0000000..29c53bd --- /dev/null +++ b/bin/password @@ -0,0 +1,5 @@ +set -euo pipefail +login="${DEFAULT_INTERNET_LOGIN:-${1-}}" +file="$(find web -type f -path "web/*/$DEFAULT_INTERNET_LOGIN.gpg" -print | fzf --delimiter='/' --with-nth=2)" +password_entry="${file%.gpg}" +pass -c "$password_entry" \ No newline at end of file From f4c62c354c02a304f03ca0bfdc7152beb00565c4 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Fri, 15 Aug 2025 09:49:16 +0200 Subject: [PATCH 42/52] Add notes --- bin/password | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/bin/password b/bin/password index 29c53bd..aa38619 100755 --- a/bin/password +++ b/bin/password @@ -1,3 +1,10 @@ +# search & copy +# password amazon +# password -l artur@gurgul.org + +# password -a artur@gurgul.org http://example.com/login + + set -euo pipefail login="${DEFAULT_INTERNET_LOGIN:-${1-}}" file="$(find web -type f -path "web/*/$DEFAULT_INTERNET_LOGIN.gpg" -print | fzf --delimiter='/' --with-nth=2)" From 73e3f7548b54491a04d9f12d9cafd23b68cd1895 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Fri, 15 Aug 2025 09:56:42 +0200 Subject: [PATCH 43/52] Add option to run vms fullscreen --- lib/vm/qemu.rb | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 905e56a..9e1d9c7 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -68,7 +68,8 @@ module Qemu cdrom: nil, detach: true, ram: 2048, - cpus: 1 + cpus: 1, + fullscreen: true } opts = defaults.merge(options) @@ -82,7 +83,12 @@ module Qemu args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] #args += ["-device", "virtio-gpu-device"] - args += ["-display", "cocoa"] + if opts[:fullscreen] + args += ["-display", "cocoa,full-screen=on"] + else + args += ["-display", "cocoa"] + end + args += ["-device", "qemu-xhci,id=xhci"] args += ["-device", "usb-kbd"] args += ["-device", "usb-tablet"] From 83a0f81cbea522f5a7eebfc9db062a5851efbe3d Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Sun, 17 Aug 2025 10:30:42 +0200 Subject: [PATCH 44/52] Add possibility to run without display --- bin/vm | 2 +- bin/vm-runner | 1 + lib/virtual-machine.rb | 2 +- lib/vm/qemu.rb | 66 ++++++++++++++++++++++++++++++------------ 4 files changed, 51 insertions(+), 20 deletions(-) create mode 100755 bin/vm-runner diff --git a/bin/vm b/bin/vm index 657bb10..ebd2c02 100755 --- a/bin/vm +++ b/bin/vm @@ -70,4 +70,4 @@ when :attach puts "if not paramteres attaches to last used" else puts "Error not found #{options.type}" -end \ No newline at end of file +end diff --git a/bin/vm-runner b/bin/vm-runner new file mode 100755 index 0000000..8569ad3 --- /dev/null +++ b/bin/vm-runner @@ -0,0 +1 @@ +[Install Zen on Linux | Flathub](https://flathub.org/apps/app.zen_browser.zen) \ No newline at end of file diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 27abeb7..436cfcc 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -129,4 +129,4 @@ module VirtualMachine # f.write("\0") end end -end \ No newline at end of file +end diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 9e1d9c7..fdc050b 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -1,6 +1,20 @@ require "fileutils" require "ostruct" +module DisplayMode + def self.none + 0 + end + + def self.fullscreen + 1 + end + + def self.window + 2 + end +end + module Qemu def self.qemu_bin_for(arch) { @@ -69,10 +83,15 @@ module Qemu detach: true, ram: 2048, cpus: 1, - fullscreen: true + display: DisplayMode::none } + + # for testing only + defaults[:detach] = false + defaults[:display] = DisplayMode.window + opts = defaults.merge(options) - + puts options puts opts qemu = qemu_bin_for(arch) @@ -82,22 +101,31 @@ module Qemu # args += ["-bios", "/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] - #args += ["-device", "virtio-gpu-device"] - if opts[:fullscreen] - args += ["-display", "cocoa,full-screen=on"] + if opts[:display] == DisplayMode::none + port = 2222 + args += ['-nographic'] + args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{port}-:22"] + #args += ['-device', 'e1000,netdev=net0'] + args += ['-device', 'virtio-net-pci,netdev=net0'] + puts "ssh -p #{port} user@localhost" + else - args += ["-display", "cocoa"] + #args += ["-device", "virtio-gpu-device"] + if opts[:display] == DisplayMode::fullscreen + args += ["-display", "cocoa,full-screen=on"] + else + args += ["-display", "cocoa"] + end + + args += ["-device", "qemu-xhci,id=xhci"] + args += ["-device", "usb-kbd"] + args += ["-device", "usb-tablet"] + + args += ["-device", "virtio-keyboard-device"] + args += ["-device", "virtio-mouse-device"] + args += ["-device", "virtio-gpu"] end - args += ["-device", "qemu-xhci,id=xhci"] - args += ["-device", "usb-kbd"] - args += ["-device", "usb-tablet"] - - args += ["-device", "virtio-keyboard-device"] - args += ["-device", "virtio-mouse-device"] - args += ["-device", "virtio-gpu"] - - # copy to /Users/artur/.cache/dat/vm/debian/arm64/debian/vars.fd # /opt/homebrew/share/qemu/edk2-aarch64-vars.fd # args += ["-drive", "if=pflash,format=raw,unit=1,file=/Users/artur/.cache/dat/vm/debian/arm64/debian/vars.fd"] @@ -111,22 +139,24 @@ module Qemu if opts[:cdrom] != nil args += ["-cdrom", opts[:cdrom]] end - args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT +# args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT # optional: uncomment to run headless with VNC on :5901 # args += ["-display", "none", "-vnc", "127.0.0.1:1"] cmd = [qemu, *args] puts "Launching: #{cmd.join(' ')}" - pid = Process.spawn(*cmd, pgroup: true, out: $stdout, err: $stderr) if opts[:detach] + log = File.open("log.txt", "w") + pid = Process.spawn(*cmd, pgroup: false, out: log, err: log) Process.detach(pid) puts "QEMU pid=#{pid}" else + pid = Process.spawn(*cmd, pgroup: true, out: $stdout, err: $stderr) Process.wait(pid) status = $? puts "Exit status: #{status.exitstatus}" end end -end \ No newline at end of file +end From 91a36e03a84b944eb6512fb7fe5ee72dbd822c22 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Sun, 17 Aug 2025 17:21:38 +0200 Subject: [PATCH 45/52] add option to detache the VM from CLI --- bin/vm | 4 ++++ lib/virtual-machine.rb | 9 +++++++-- lib/vm/qemu.rb | 21 +++++++++++++++++++-- recipes/qemu/macos.yml | 28 ++++++++++++++++++++++++++++ 4 files changed, 58 insertions(+), 4 deletions(-) create mode 100644 recipes/qemu/macos.yml diff --git a/bin/vm b/bin/vm index ebd2c02..609384e 100755 --- a/bin/vm +++ b/bin/vm @@ -22,6 +22,10 @@ OptionParser.new do |opt| opt.on('--name NAME', 'Virtaul Machine name') do |name| options.name = name end + + opt.on('--detached') do + options.detached = true + end # --port (ssh port) # --attached (do not leave the shell process) # new from copy diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 436cfcc..ccb8606 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -58,6 +58,10 @@ module VirtualMachine else options[:arch] = System.arch_to_symbol(options[:arch]) end + + if options[:detached] == nil + options[:detached] = false + end end def self.archive(options) @@ -90,7 +94,7 @@ module VirtualMachine options[:arch], disk_img_path, cpus: [1, System.cpus - 2].max, - detach: true + detach: options[:detached] ) end @@ -109,7 +113,8 @@ module VirtualMachine disk_img_path, cpus: [1, System.cpus - 2].max, cdrom: path, - detach: true + detach: options[:detached], + display: DisplayMode.window ) end end diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index fdc050b..cad148a 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -88,7 +88,7 @@ module Qemu # for testing only defaults[:detach] = false - defaults[:display] = DisplayMode.window + defaults[:display] = DisplayMode.fullscreen opts = defaults.merge(options) puts options @@ -112,7 +112,17 @@ module Qemu else #args += ["-device", "virtio-gpu-device"] if opts[:display] == DisplayMode::fullscreen - args += ["-display", "cocoa,full-screen=on"] + #args += ["-display", "cocoa,full-screen=on"] + + # attempts: + #args += ["-display", "cocoa,full-screen=on,retina=on"] + # brew install gtk+3 sdl2 + # args += ["-display", "sdl,gl=on,full-screen=on"] + args += ["-device", "virtio-gpu-gl-pci"] + args += ["-display", "sdl,gl=on,full-screen=on"] + #args += ["-display", "gtk,gl=on,full-screen=on"] + #args += ["-display", "cocoa,full-screen=on"] + #args += ["-display", "cocoa,gl=es,full-screen=on"] else args += ["-display", "cocoa"] end @@ -124,6 +134,13 @@ module Qemu args += ["-device", "virtio-keyboard-device"] args += ["-device", "virtio-mouse-device"] args += ["-device", "virtio-gpu"] + + + #args += ['-nic', 'user,model=virtio-net-pci'] + args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] + # macOS vmnet (shares Mac’s LAN) + # -netdev vmnet-shared,id=n1 \ + # -device virtio-net-pci,netdev=n1 end # copy to /Users/artur/.cache/dat/vm/debian/arm64/debian/vars.fd diff --git a/recipes/qemu/macos.yml b/recipes/qemu/macos.yml new file mode 100644 index 0000000..4a902dd --- /dev/null +++ b/recipes/qemu/macos.yml @@ -0,0 +1,28 @@ +packages: + - meson + - ninja + - pkg-config + - glib + - pixman + - sdl2 + - libepoxy + - libslirp + - gettext + + +# pkg-config --modversion sdl2 # should print a version +# pkg-config --modversion epoxy # should print a version + +repository: + url: https://gitlab.com/qemu-project/qemu.git + branch: v10.0.3 + +steps: + - mkdir build + - cd build + - | + ../configure \ + --enable-sdl \ + --enable-opengl \ + --target-list=aarch64-softmmu,x86_64-softmmu \ + --prefix="$HOME/.local/qemu-sdl" \ No newline at end of file From c8b057e6a229fd5c1ce130dacb4983de4bee5c4a Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 26 Aug 2025 08:51:37 +0200 Subject: [PATCH 46/52] save --- bin/password | 22 +++++++++++++++------- bin/zshrc/init | 2 +- install | 3 +++ lib/vm/qemu.rb | 24 ++++++++++++++---------- 4 files changed, 33 insertions(+), 18 deletions(-) diff --git a/bin/password b/bin/password index aa38619..c94eb34 100755 --- a/bin/password +++ b/bin/password @@ -1,12 +1,20 @@ -# search & copy +#!/usr/bin/env ruby + +# password for account, must be one or the default # password amazon +# copy login +# password amazon -l + +# search for password for given account # password -l artur@gurgul.org -# password -a artur@gurgul.org http://example.com/login +# http://example.com/login +# password example -l artur@gurgul.org -set -euo pipefail -login="${DEFAULT_INTERNET_LOGIN:-${1-}}" -file="$(find web -type f -path "web/*/$DEFAULT_INTERNET_LOGIN.gpg" -print | fzf --delimiter='/' --with-nth=2)" -password_entry="${file%.gpg}" -pass -c "$password_entry" \ No newline at end of file + +# set -euo pipefail +# login="${DEFAULT_INTERNET_LOGIN:-${1-}}" +# file="$(find web -type f -path "web/*/$DEFAULT_INTERNET_LOGIN.gpg" -print | fzf --delimiter='/' --with-nth=2)" +# password_entry="${file%.gpg}" +# pass -c "$password_entry" \ No newline at end of file diff --git a/bin/zshrc/init b/bin/zshrc/init index 79c3372..39fd3e4 100644 --- a/bin/zshrc/init +++ b/bin/zshrc/init @@ -11,7 +11,7 @@ export RUBYLIB="$DAT_ROOT/lib" export PASSWORD_STORE_DIR=$HOME/.local/lib/secure-vault/passwords export NOTES_DIR=$HOME/.local/lib/notes -path=("$GEM_HOME/bin" $path) +path=("$GEM_HOME/bin" "$HOME/.cargo/bin" $path) alias gf='git log --all --oneline | fzf' diff --git a/install b/install index 7dcfed1..c8f70f9 100755 --- a/install +++ b/install @@ -1,5 +1,8 @@ #!/usr/bin/env bash +## Software for bootstraping +# curl, gcc + # TODO: Packages that need to be added # gem install ruby-lsp diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index cad148a..5d4fd15 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -88,7 +88,10 @@ module Qemu # for testing only defaults[:detach] = false + defaults[:display] = DisplayMode.fullscreen + defaults[:display] = DisplayMode.window + defaults[:display] = DisplayMode.none opts = defaults.merge(options) puts options @@ -112,17 +115,19 @@ module Qemu else #args += ["-device", "virtio-gpu-device"] if opts[:display] == DisplayMode::fullscreen - #args += ["-display", "cocoa,full-screen=on"] + # #args += ["-display", "cocoa,full-screen=on"] + # # attempts: + # #args += ["-display", "cocoa,full-screen=on,retina=on"] + # # brew install gtk+3 sdl2 + # # args += ["-display", "sdl,gl=on,full-screen=on"] + # #args += ["-display", "gtk,gl=on,full-screen=on"] + # #args += ["-display", "cocoa,full-screen=on"] + # #args += ["-display", "cocoa,gl=es,full-screen=on"] - # attempts: - #args += ["-display", "cocoa,full-screen=on,retina=on"] - # brew install gtk+3 sdl2 + #### TODO: try make it work with custom build + # args += ["-device", "virtio-gpu-gl-pci"] # args += ["-display", "sdl,gl=on,full-screen=on"] - args += ["-device", "virtio-gpu-gl-pci"] - args += ["-display", "sdl,gl=on,full-screen=on"] - #args += ["-display", "gtk,gl=on,full-screen=on"] - #args += ["-display", "cocoa,full-screen=on"] - #args += ["-display", "cocoa,gl=es,full-screen=on"] + args += ["-display", "cocoa,full-screen=on"] else args += ["-display", "cocoa"] end @@ -135,7 +140,6 @@ module Qemu args += ["-device", "virtio-mouse-device"] args += ["-device", "virtio-gpu"] - #args += ['-nic', 'user,model=virtio-net-pci'] args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # macOS vmnet (shares Mac’s LAN) From 54d4dfcb33b3b92b163b6a9080373c961520a411 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 26 Aug 2025 08:52:28 +0200 Subject: [PATCH 47/52] save --- bin/vm | 8 ++++++++ lib/virtual-machine.rb | 28 +++++++++++++++++++++++----- lib/vm/qemu.rb | 40 +++++++++++++++++++++++++++++++++++++--- 3 files changed, 68 insertions(+), 8 deletions(-) diff --git a/bin/vm b/bin/vm index 609384e..322c850 100755 --- a/bin/vm +++ b/bin/vm @@ -23,6 +23,14 @@ OptionParser.new do |opt| options.name = name end + opt.on('--cdrom CDROM', 'Use local cdrom image') do |cdrom| + options.cdrom = cdrom + end + + opt.on('--tpm', 'Use tpm') do + options.tpm = true + end + opt.on('--detached') do options.detached = true end diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index ccb8606..874a9fe 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -62,6 +62,10 @@ module VirtualMachine if options[:detached] == nil options[:detached] = false end + + if options[:tpm] == nil + options[:tpm] = false + end end def self.archive(options) @@ -98,14 +102,16 @@ module VirtualMachine ) end + # vm setup windows --arch arm64 --iso /Users/agurgul/Downloads/win.iso --tpm + # vm setup windows --tpm --cdrom /Users/agurgul/Downloads/win.iso def self.setup(options) fill_defaults(options) - url = distro(options[:name], options[:arch], :install) - disk_img_path = root_img_path(options) + # puts options + # exit -1 - Downloader.get(url) do |path| - + get_cdrom_image(options) do |path| + disk_img_path = root_img_path(options) create_disk_image(disk_img_path, 64000) Qemu.launch( @@ -114,11 +120,23 @@ module VirtualMachine cpus: [1, System.cpus - 2].max, cdrom: path, detach: options[:detached], - display: DisplayMode.window + display: DisplayMode.window, + tpm: options[:tpm] ) end end + def self.get_cdrom_image(options) + if options[:cdrom] == nil + url = distro(options[:name], options[:arch], :install) + Downloader.get(url) do |path| + yield path + end + else + yield options[:cdrom] + end + end + # size in MB # lsof /Users/artur/.cache/dat/vm/debian/arm64/debian/root.img def self.create_disk_image(path, size) diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 5d4fd15..50140d3 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -81,7 +81,7 @@ module Qemu arch: System::ARCH, cdrom: nil, detach: true, - ram: 2048, + ram: 2048 * 8, cpus: 1, display: DisplayMode::none } @@ -102,7 +102,9 @@ module Qemu if System::OS == :macos && arch == :arm64 # args += ["-bios", "/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] + # cp /opt/homebrew/share/qemu/edk2-arm-vars.fd ~/edk2-arm-vars.fd args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] + args += ["-drive", "if=pflash,format=raw,unit=1,file=/Users/agurgul/edk2-arm-vars.fd"] if opts[:display] == DisplayMode::none port = 2222 @@ -156,10 +158,40 @@ module Qemu args += machine_args_for(arch) args += ["-m", opts[:ram].to_s, "-smp", opts[:cpus].to_s] args += ["-name", "FirstVM", "-boot", "order=d"] # boot from CD first - args += ["-drive", "file=#{disk_path},if=virtio,cache=writeback,format=raw"] + args += ["-drive", "file=#{disk_path},if=virtio,cache=writeback,format=raw,id=nvme0"] + #args += ["-device", "nvme,serial=nvme0,drive=nvme0,bootindex=2"] + if opts[:cdrom] != nil - args += ["-cdrom", opts[:cdrom]] + #args += ["-cdrom", opts[:cdrom]] + + # args += ["-device", "virtio-scsi-pci,id=scsi"] + # args += ["-drive", "if=none,id=cd,format=raw,file=#{opts[:cdrom]},media=cdrom"] + # args += ["-device", "scsi-cd,drive=cd,bootindex=1"] + + + args += ["-drive", "id=cd,format=raw,file=#{opts[:cdrom]},media=cdrom"] + args += ["-device", "usb-storage,drive=cd,bootindex=1"] + + args += ["-device", "ramfb"] + # args += ["-device", "virtio-gpu-pci"] + # args += ["-display", "default,show-cursor=on"] end + + if opts[:tpm] + # brew install swtpm + # swtpm socket --tpm2 --ctrl type=unixio,path=./tpm/tpm.sock --tpmstate dir=./tpm --daemon + + # args += ["-chardev", "socket,id=chrtpm,path=/Users/agurgul/Downloads/tpm/tpm.sock"] + # args += ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"] + # args += ["-device", "tpm-crb-device,tpmdev=tpm0"] + + args += ["-chardev", "socket,id=chrtpm,path=/Users/agurgul/Downloads/tpm/tpm.sock"] + args += ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"] + #args += ["-device", "tpm-tis,tpmdev=tpm0"] + args += ["-device", "tpm-tis-device,tpmdev=tpm0"] + # nic user,ipv6=off,model=rtl8139,mac=84:1b:77:c9:03:a6 + end + # args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT # optional: uncomment to run headless with VNC on :5901 # args += ["-display", "none", "-vnc", "127.0.0.1:1"] @@ -181,3 +213,5 @@ module Qemu end end end + + From 605cfd38f9cffd62ddef0bd9d71ce19ded28bcca Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Tue, 26 Aug 2025 17:50:38 +0200 Subject: [PATCH 48/52] save --- bin/vm-cmd | 49 ++++++++++++++++++++++++++++++++++++++++++ lib/system.rb | 7 ++++++ lib/system/debian.rb | 8 +++++++ lib/system/macos.rb | 19 ++++++++++++++++ lib/user.rb | 2 +- lib/virtual-machine.rb | 15 ++++++++++++- lib/vm/qemu.rb | 27 +++++++++++++++++++---- readme.md | 9 ++++++++ 8 files changed, 130 insertions(+), 6 deletions(-) create mode 100755 bin/vm-cmd diff --git a/bin/vm-cmd b/bin/vm-cmd new file mode 100755 index 0000000..41874eb --- /dev/null +++ b/bin/vm-cmd @@ -0,0 +1,49 @@ +#!/usr/bin/env bash + +# brew install qemu swtpm + +# qemu-system-aarch64 -drive if=pflash,format=raw,unit=0,readonly=on,file=/Volumes/Cache/vms/image/win11-arm64/win11/code.fd -drive if=pflash,format=raw,unit=1,file=/Volumes/Cache/vms/image/win11-arm64/win11/vars.fd -display cocoa -device qemu-xhci,id=xhci -device usb-kbd -device usb-tablet -device virtio-keyboard-device -device virtio-mouse-device -device virtio-gpu -device virtio-net,netdev=n0 -netdev user,id=n0 -accel hvf -machine virt -cpu max -m 16384 -smp 6 -name FirstVM -boot order=d -drive file=/Volumes/Cache/vms/image/win11-arm64/win11/root.img,if=virtio,cache=writeback,format=raw,id=nvme0 -drive id=cd,format=raw,file=/Volumes/Cache/downloads/win11arm64.iso,media=cdrom -device usb-storage,drive=cd,bootindex=1 -device ramfb -chardev socket,id=chrtpm,path=/Users/agurgul/Downloads/tpm/tpm.sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis-device,tpmdev=tpm0 +# vm setup win11 --tpm --cdrom /Volumes/Cache/downloads/win11arm64.iso + + +# qemu-system-aarch64 -drive if=pflash,format=raw,unit=0,readonly=on,file=/Volumes/Cache/vms/image/win11-arm64/win11/code.fd -drive if=pflash,format=raw,unit=1,file=/Volumes/Cache/vms/image/win11-arm64/win11/vars.fd -display cocoa -device qemu-xhci,id=xhci -device usb-kbd -device usb-tablet -device virtio-keyboard-device -device virtio-mouse-device -device virtio-gpu -device virtio-net,netdev=n0 -netdev user,id=n0 -accel hvf -machine virt -cpu max -m 16384 -smp 6 -name FirstVM -boot order=d -drive file=/Volumes/Cache/vms/image/win11-arm64/win11/root.img,if=virtio,cache=writeback,format=raw,id=nvme0 -drive id=cd,format=raw,file=/Volumes/Cache/downloads/win11arm64.iso,media=cdrom -device usb-storage,drive=cd,bootindex=1 -device ramfb -chardev socket,id=chrtpm,path=/Users/agurgul/Downloads/tpm/tpm.sock -tpmdev emulator,id=tpm0,chardev=chrtpm -device tpm-tis-device,tpmdev=tpm0 + +# -netdev user,id=net0,hostfwd=tcp::33890-:3389,hostfwd=tcp::59850-:5985 \ +# -machine virt,accel=hvf,highmem=off \ + + +# -device virtio-keyboard-pci \ +# -device virtio-tablet-pci \ + + +qemu-system-aarch64 \ + -boot order=d \ + -machine virt,accel=hvf \ + -cpu host -smp 6 -m 8G \ + -drive if=pflash,format=raw,file=/Volumes/Cache/vms/image/win11-arm64/win11/code.fd,readonly=on \ + -drive if=pflash,format=raw,file=/Volumes/Cache/vms/image/win11-arm64/win11/vars.fd \ + -drive file=/Volumes/Cache/vms/image/win11-arm64/win11/root.img,if=none,format=raw,id=drv0 \ + -device ich9-ahci,id=ahci0 \ + -device nvme,drive=drv0,serial=nvme0 \ + -device virtio-scsi-pci,id=scsi0 \ + -drive file=/Volumes/Cache/downloads/win11arm64.iso,if=none,media=cdrom,id=cd0 \ + -device scsi-cd,drive=cd0,bus=scsi0.0,bootindex=1 \ + -device ramfb \ + -device qemu-xhci,id=xhci \ + -device usb-kbd,bus=xhci.0,port=1 \ + -device usb-tablet,bus=xhci.0,port=2 \ + -netdev user,id=net0,hostfwd=tcp::33890-:3389 \ + -device virtio-net-pci,netdev=net0 \ + -chardev socket,id=chrtpm,path=/Users/artur/Downloads/tpm/tpm.sock \ + -tpmdev emulator,id=tpm0,chardev=chrtpm \ + -device tpm-tis-device,tpmdev=tpm0 + + +# in Shell +# FS0: +# cd EFI\BOOT +# BOOTAA64.EFI + + +# Disk driver: https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/ +# Used: https://fedorapeople.org/groups/virt/virtio-win/direct-downloads/archive-virtio/virtio-win-0.1.271-1/ \ No newline at end of file diff --git a/lib/system.rb b/lib/system.rb index 28d7f38..bbedf0b 100644 --- a/lib/system.rb +++ b/lib/system.rb @@ -37,6 +37,13 @@ module System puts os_name end + def self.qemu_paths + { + code_fd: qemu_code_fd_path, + vars_fd: qemu_vars_fd_path + } + end + def self.arch_to_symbol(arch) normalize_architecture_string(arch) end diff --git a/lib/system/debian.rb b/lib/system/debian.rb index c7803df..e021a1b 100644 --- a/lib/system/debian.rb +++ b/lib/system/debian.rb @@ -54,4 +54,12 @@ module DebianSystem puts "Failed to uninstall some packages." end end + + def qemu_code_fd_path() + raise "not supported yet" + end + + def qemu_vars_fd_path() + raise "not supported yet" + end end diff --git a/lib/system/macos.rb b/lib/system/macos.rb index cc3c9ff..b1275ee 100644 --- a/lib/system/macos.rb +++ b/lib/system/macos.rb @@ -54,4 +54,23 @@ module MacOSSystem puts "Failed to uninstall some packages." end end + + def qemu_code_fd_path() + case arch_to_symbol(arch) + when :arm64 + "/opt/homebrew/share/qemu/edk2-aarch64-code.fd" + else + raise "not supported yet" + end + end + + def qemu_vars_fd_path() + case arch_to_symbol(arch) + when :arm64 + "/opt/homebrew/share/qemu/edk2-arm-vars.fd" + else + raise "not supported yet" + end + end + end diff --git a/lib/user.rb b/lib/user.rb index 724436b..903b44a 100644 --- a/lib/user.rb +++ b/lib/user.rb @@ -36,6 +36,6 @@ module User def self.cache_path - "#{ENV["HOME"]}/.cache/dat/" + ENV["DAT_CACHE_PATH"] || "#{ENV["HOME"]}/.cache/dat/" end end diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 874a9fe..20c2ca7 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -66,6 +66,14 @@ module VirtualMachine if options[:tpm] == nil options[:tpm] = false end + + unless options[:vars_fd] + options[:vars_fd] = File.join(vm_dir(options), "vars.fd") + end + + unless options[:code_fd] + options[:code_fd] = File.join(vm_dir(options), "code.fd") + end end def self.archive(options) @@ -97,8 +105,11 @@ module VirtualMachine Qemu.launch( options[:arch], disk_img_path, + code_fd: options[:code_fd], + vars_fd: options[:vars_fd], cpus: [1, System.cpus - 2].max, - detach: options[:detached] + detach: options[:detached], + tpm: options[:tpm] ) end @@ -117,6 +128,8 @@ module VirtualMachine Qemu.launch( options[:arch], disk_img_path, + code_fd: options[:code_fd], + vars_fd: options[:vars_fd], cpus: [1, System.cpus - 2].max, cdrom: path, detach: options[:detached], diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 50140d3..60aeb0f 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -91,20 +91,33 @@ module Qemu defaults[:display] = DisplayMode.fullscreen defaults[:display] = DisplayMode.window - defaults[:display] = DisplayMode.none + # defaults[:display] = DisplayMode.none opts = defaults.merge(options) + puts options puts opts qemu = qemu_bin_for(arch) args = [] - + if System::OS == :macos && arch == :arm64 # args += ["-bios", "/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] # cp /opt/homebrew/share/qemu/edk2-arm-vars.fd ~/edk2-arm-vars.fd - args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=/opt/homebrew/share/qemu/edk2-aarch64-code.fd"] - args += ["-drive", "if=pflash,format=raw,unit=1,file=/Users/agurgul/edk2-arm-vars.fd"] + + + unless File.exist?(opts[:vars_fd]) + #System.qemu_paths + FileUtils.cp(System.qemu_vars_fd_path, opts[:vars_fd]) + + end + + unless File.exist?(opts[:code_fd]) + FileUtils.cp(System.qemu_code_fd_path, opts[:code_fd]) + end + + args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=#{opts[:code_fd]}"] + args += ["-drive", "if=pflash,format=raw,unit=1,file=#{opts[:vars_fd]}"] if opts[:display] == DisplayMode::none port = 2222 @@ -181,6 +194,9 @@ module Qemu # brew install swtpm # swtpm socket --tpm2 --ctrl type=unixio,path=./tpm/tpm.sock --tpmstate dir=./tpm --daemon + + ["swtpm", "socket", "--tpm2", "--ctrl", "type=unixio,path=./tpm/tpm.sock", "--tpmstate", "dir=./tpm"] + # args += ["-chardev", "socket,id=chrtpm,path=/Users/agurgul/Downloads/tpm/tpm.sock"] # args += ["-tpmdev", "emulator,id=tpm0,chardev=chrtpm"] # args += ["-device", "tpm-crb-device,tpmdev=tpm0"] @@ -190,6 +206,9 @@ module Qemu #args += ["-device", "tpm-tis,tpmdev=tpm0"] args += ["-device", "tpm-tis-device,tpmdev=tpm0"] # nic user,ipv6=off,model=rtl8139,mac=84:1b:77:c9:03:a6 + + # TODO: Shared network on macOS + # -netdev vmnet-shared,id=net0 end # args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT diff --git a/readme.md b/readme.md index 0403984..3d643d0 100644 --- a/readme.md +++ b/readme.md @@ -4,3 +4,12 @@ ```bash curl -sSL https://gurgul.pro/artur/environment/raw/branch/main/install | bash ``` + + +Config: + +| Variable Name | Default Value | Description | +|---------------------------|--------------------------------|-------------| +| `DAT_CACHE_PATH` | `${HOME}/.cache/dat/` | Path where cached data is stored. | +| `DEFAULT_INTERNET_LOGIN` | not set | | +| `DAT_VM_DATA` | `${DAT_CACHE_PATH}/vm` | Directory for storing VM-related data. | \ No newline at end of file From 764967c8e0889b8e4db2557d49f0d6fe48398835 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Wed, 27 Aug 2025 11:37:46 +0200 Subject: [PATCH 49/52] Trying to install Windows11 ARM on QEMU --- bin/recipes/edk2/make | 108 +++++++++ .../microsoft corporation kek 2k ca 2023.crt | Bin 0 -> 1462 bytes bin/recipes/edk2/notes.md | 61 +++++ bin/recipes/edk2/windows uefi ca 2023.crt | Bin 0 -> 1454 bytes bin/vm-cmd | 217 ++++++++++++++++-- lib/virtual-machine.rb | 2 +- 6 files changed, 365 insertions(+), 23 deletions(-) create mode 100755 bin/recipes/edk2/make create mode 100644 bin/recipes/edk2/microsoft corporation kek 2k ca 2023.crt create mode 100644 bin/recipes/edk2/notes.md create mode 100644 bin/recipes/edk2/windows uefi ca 2023.crt diff --git a/bin/recipes/edk2/make b/bin/recipes/edk2/make new file mode 100755 index 0000000..83e754b --- /dev/null +++ b/bin/recipes/edk2/make @@ -0,0 +1,108 @@ +#!/usr/bin/env bash + +rm -rf edk2 +rm -rf output + +git clone https://github.com/tianocore/edk2.git +cd edk2 +git checkout fc0fffa7e9089e7b79b9ae7babb950f1f153e0ae + + +# 0) Make sure LLVM is installed +brew install llvm acpica nasm +brew install lld + +# 1) Point PATH and tool vars to Homebrew’s LLVM (Apple Silicon path shown) +export LLVM_PREFIX="/opt/homebrew/opt/llvm" # Intel Macs: /usr/local/opt/llvm +export LLD_PREFIX="/opt/homebrew/opt/lld" +export PATH="$LLVM_PREFIX/bin:$LLD_PREFIX/bin:$PATH" + +hash -r + +# 2) Explicitly select LLVM tools so EDK2 doesn’t fall back to Apple’s +export CC="$LLVM_PREFIX/bin/clang" +export CXX="$LLVM_PREFIX/bin/clang++" +export LD="$LLD_PREFIX/bin/ld.lld" +export AR="$LLVM_PREFIX/bin/llvm-ar" +export RANLIB="$LLVM_PREFIX/bin/llvm-ranlib" +export NM="$LLVM_PREFIX/bin/llvm-nm" +export STRIP="$LLVM_PREFIX/bin/llvm-strip" +export OBJCOPY="$LLVM_PREFIX/bin/llvm-objcopy" +export OBJDUMP="$LLVM_PREFIX/bin/llvm-objdump" + +# 3) Sanity check — these MUST point into .../opt/llvm/bin +which clang; clang --version +which ld.lld +which llvm-ar + +# 4) Rebuild tools & firmware +make -C BaseTools -j +source ./edksetup.sh + +build -a AARCH64 -t CLANGDWARF \ + -p ShellPkg/ShellPkg.dsc \ + -m ShellPkg/Application/KeyTools/KeyTool/KeyTool.inf \ + -b RELEASE + + +cd .. + +mkdir -p output/keys/EFI/Boot +openssl x509 -in "microsoft corporation kek 2k ca 2023.crt" -outform DER -out output/keys/kek2023.cer +openssl x509 -in "windows uefi ca 2023.crt" -outform DER -out output/keys/db2023.cer + +open /Users/artur/projs/edk2 + +exit 0 + + +# (Optional) clean the previous failed build to avoid stale flags/objects +rm -rf Build/ArmVirtQemu-AARCH64 + +build -a AARCH64 \ + -t CLANGDWARF \ + -p ArmVirtPkg/ArmVirtQemu.dsc \ + -D SECURE_BOOT_ENABLE=TRUE \ + -b DEBUG + + + +# CODE="/Volumes/Cache/vms/image/win11-arm64/win11/QEMU_EFI.fd" +# VARS="/Volumes/Cache/vms/image/win11-arm64/win11/QEMU_VARS.fd" + + +# # Make blank 64 MiB raws +# qemu-img create -f raw QEMU_EFI-pflash.raw 64M +# qemu-img create -f raw QEMU_VARS-pflash.raw 64M + +# # Copy firmware into the front of each file without truncating the 64 MiB size +# dd if="$CODE" of=QEMU_EFI-pflash.raw conv=notrunc +# dd if="$VARS" of=QEMU_VARS-pflash.raw conv=notrunc + +# # Confirm size is exactly 67108864 bytes +# stat -f "%z %N" QEMU_EFI-pflash.raw QEMU_VARS-pflash.raw + + + +# https://learn.microsoft.com/en-us/windows-hardware/manufacture/desktop/windows-secure-boot-key-creation-and-management-guidance?view=windows-11 +# https://go.microsoft.com/fwlink/?linkid=2239775 +# https://go.microsoft.com/fwlink/?linkid=2239776 + + +# -drive if=none,id=mskeys,format=raw,file=fat:rw:keys +# -device usb-storage,drive=mskeys + + +# 5. Enroll the certs in firmware +# Boot into UEFI shell, run FS0:\EFI\Boot\KeyTool.efi. + +# In KeyTool: +# Go to Edit Keys → KEK → Add New Key, pick kek2023.cer. +# Go to Edit Keys → db → Add New Key, pick db2023.cer. +# If no PK is present yet, you must enroll a Platform Key (PK) first (can be self-signed with OpenSSL; KeyTool can also generate). Without a PK, Secure Boot remains “greyed out.” + +# ✅ After PK + KEK + DB are enrolled, reboot → go into firmware setup → Secure Boot option will be toggleable → set it to Enabled. +# Now Windows 11 installer/bootloader should run under Secure Boot. +# Ref: 68adf715-f29c-832e-89d7-a37025f63cf3 + + diff --git a/bin/recipes/edk2/microsoft corporation kek 2k ca 2023.crt b/bin/recipes/edk2/microsoft corporation kek 2k ca 2023.crt new file mode 100644 index 0000000000000000000000000000000000000000..e6ffb4f975fae9658631e5ca7697b63d3692b869 GIT binary patch literal 1462 zcmXqLV%=oW#5`*OGZP~dldv%Z1B0-L*p9^9rY3J78^|-@W#iOp^Jx3d%gD&h%3u&> z$Zf#M#vIDRCd?EXY$#_S1LANAi~43J7v&e{r^7bTWt=I0q|8)$;$nT3@Q z@;@`l=OY|zB0gzQ;HRtDxKMt%mMI2ThBBO}A(xQ^~(ox@y5 zUVc5-sQZ!sqnfGtMbF^EjnR{X);|=zf3Mqe?|Prq(%`4JxQu4#8>?;8-Zy3D{5Q7_ z^|6{v66R0dU$UrZg=XD+BSlS)$Rp)Pv?lqqZ(}xpJ?qiolUqC#4<}A}B_6o(xyU1x zWWB4OjxGJPC{8}}SRn6m&EG*>vmbZ7$oRke*QCt#N5uZeK8=5H|4mvbcjBQn)(Uy| zxA4B!K5^lYiC@z+%Xc3QTUl)1Un~24_|(CxJ^I`N8nZoSZ4eMya&yYljOep@&#T_- zS>U;2W$+DO{Y}*;ZtdCh{LktFf%2)Yo`2U~vKK#{HtSQZ*iLa zh_Q$~SY5Pdn%(~$X|rnuqMYqwoI={;4J6?jM3`83GP0x@q!=W@_y$aEVFYbVEh;G{)y@*+Bnixa%uT?cZDnE; z`jx)U-*V*+lRL3m6_V8!@8&f1OFy!)HR6ifAHnuK|3|OH3h9=wvrnz<=5XOt>(#jt zpQo(K{wvx?O|Q(*SugL!3B!Y4mp(CVU-q%|$+Wj@k4{aka9Lqoc17Fw`kp)O)~kIr z*G}u)e)UlQYPZ9&oBzIDI`3=QcJ0>xGZxOzmYHq${T-k9;=gA456^1Mka3u)c8a-o zndz%bT+6$~)+sVF$f@&h=iB6D7^e5^N8c+RlMjCP3k)(@E}p+UC&exQjN;kT?^%s@ zb1b_t-_;`4@6wz}oiB@@RB@Qf`7b-PE8A|ne29tXuPs%lZ!M6IY}D5J)}glHSI(!` z2XCx%e3<`dzwnDZk>}5kGj}`86tG~+E!fL=;$ClU(F%s|KrcGzutFc-y5g~ST4T*&wl5x*VCq7IQx@D z?KiK7B*%@1lMeSIv21gk;J4vjRy z?NuKN{n`=}s&d1QM|0XOdVhs)-lWU&Gr!Hu?nFpB^H literal 0 HcmV?d00001 diff --git a/bin/recipes/edk2/notes.md b/bin/recipes/edk2/notes.md new file mode 100644 index 0000000..cbb8a67 --- /dev/null +++ b/bin/recipes/edk2/notes.md @@ -0,0 +1,61 @@ +That error means your **WORKSPACE** points to a *different* edk2 tree: + +``` +/Users/artur/.dat/bin/recipes/edk2/edk2 +``` + +so `build` is looking for `ShellPkg/.../KeyTool.inf` **there**, not in `~/projs/edk2`. + +Fix by resetting your env and sourcing `edksetup.sh` from the repo you actually want to build. + +## Do this exactly + +```bash +# 0) Go to the edk2 you want to use +cd ~/projs/edk2 +git rev-parse --show-toplevel # should print /Users/artur/projs/edk2 + +# 1) Clear any stale EDK2 env from previous shells +unset WORKSPACE EDK_TOOLS_PATH PACKAGES_PATH + +# 2) Rebuild BaseTools (with your LLVM/lld PATH still set like earlier) +make -C BaseTools -j + +# 3) Reset Conf inside THIS tree and re-source +rm -f Conf/target.txt Conf/tools_def.txt +source ./edksetup.sh + +# Sanity checks — these must point into ~/projs/edk2 +echo "WORKSPACE=$WORKSPACE" +echo "EDK_TOOLS_PATH=$EDK_TOOLS_PATH" +test -f "$WORKSPACE/ShellPkg/Application/KeyTools/KeyTool/KeyTool.inf" && echo "KeyTool found" + +# 4) Build KeyTool for AArch64 with clang/lld +build -a AARCH64 -t CLANGDWARF \ + -p ShellPkg/ShellPkg.dsc \ + -m ShellPkg/Application/KeyTools/KeyTool/KeyTool.inf \ + -b RELEASE +``` + +### If you still see it picking the wrong path + +You probably have these variables exported in your shell config. Force them for this shell: + +```bash +export WORKSPACE="$PWD" +export EDK_TOOLS_PATH="$WORKSPACE/BaseTools" +export PACKAGES_PATH="$WORKSPACE" +source ./edksetup.sh +``` + +Then re-run the `build` command above. + +### Where the binary will land + +Typical output path (adjust for your build target): + +``` +Build/Shell/RELEASE_CLANGDWARF/AARCH64/ShellPkg/Application/KeyTools/KeyTool/KeyTool/OUTPUT/KeyTool.efi +``` + +Once you’ve got `KeyTool.efi`, mount it along with your `kek2023.cer` and `db2023.cer`, enroll **PK → KEK → db**, reboot into the firmware UI, and enable **Secure Boot**. diff --git a/bin/recipes/edk2/windows uefi ca 2023.crt b/bin/recipes/edk2/windows uefi ca 2023.crt new file mode 100644 index 0000000000000000000000000000000000000000..4c5430b28340de623d877bf09ce5284a7c45785b GIT binary patch literal 1454 zcmXqLVqIm>#5`#MGZP~dldv%Z1A|mY_Y8(GrIv$0Hjrn)%f_kI=F#?@mywa1mBFB~ z!;ss6lZ`o)g-w_#G}utsKoG>?5atR`EY8TxOE1aKGZZl30|~MVvj?T7112aQoLkm+2Bg-gpUSm@**U|vQHSj@oDIq7w8c0K& zAQGOLmy%yztPtwz=BeQ92(%4oPZOgOa#%33GB7tW@-qO%xtN+585#C$GQAj5xuyS1 zqgLK#p=*<;8`N0D?5~_8$;GeB^>6=!6*r9RFRjkcZ#?KLp6PnNwLyJx&1S1- zQ<&~*{+a{+llvk|9oyD$3UGc3ObyujRdKVLW$&sZ6F1*klcLOa)-;~i$#afB@41HN zrpSNyyGvHZom+60>{5BKO^IR78YRAYBLaELsCYKcJ1JffgAp(;jF!jZQ(w(k>nXZws z9#D}%6dMQ7SFEh;jEpRP20jK}Funm(n;q1O^73+QRsfSVKC?@Z(;6_}F*gCjZ2rk@ z|MMQsu@3$AuiNr&B1`^-waI72%e+=NclQ1`_Ngas!ow#SOGV%8xSYV_5H9~a&a^1; z-gLv9$^~D=)_KdkW#oDPJ)GakKqB+U!o_;iC+*lL{ZG3s&dEuzzpub|!t%e2$)8g8 z&wf+ZXzgHB2=}aN3StwS-@J!Un|^P^Eaj!AI^U4@IouIUq#@dW?rvY$Gg7z zw8>2Q(eqe(&NRxbogVU0La#1p!k)l;T72u$UhPTATQD)@iN4T^JyXJ{a;|uKZsmv{d%y-8GB&3!A=A{vWe2ab4;sk;b=`QM1}#B~+i@ z>|bhfYS+T|i@q&ta9!A~-gx$ExrN;+zq=Z*B6b~Jpn0#Yr{m95^}xHUFJFk}EkB;q zAA0@DBEIM+CT=2PvX5o1<<4B|^QN#^fAzUdTQu*58U$Nf8FhbnHK#+!nl(dm|M~VH zf#27J6FxhpeBlUhJUv%%Y2TNp>R%UIy?J8mwf)@B4&M2DPvu_n+7nUMXT&lk@9vlL zx$e1#u4mn8FMPFR|NHdmb9NkWFw@eCJb8d;xyyp$tv%;0qSpP`-8S`u%i6Rm Date: Thu, 28 Aug 2025 17:48:34 +0200 Subject: [PATCH 50/52] save --- bin/recipes/gnome/notes.md | 1 + bin/recipes/net/net-up | 20 ++++++++ bin/recipes/net/notes.md | 16 +++++++ bin/recipes/vnc/notes.md | 93 ++++++++++++++++++++++++++++++++++++ bin/vm | 5 ++ lib/system.rb | 22 +++++++++ lib/virtual-machine.rb | 3 +- lib/vm/qemu.rb | 98 ++++++++++++++++++++++++++++++++++++-- recipes/vnc-viewer.yml | 45 +++++++++++++++++ 9 files changed, 299 insertions(+), 4 deletions(-) create mode 100644 bin/recipes/gnome/notes.md create mode 100755 bin/recipes/net/net-up create mode 100644 bin/recipes/net/notes.md create mode 100644 bin/recipes/vnc/notes.md create mode 100644 recipes/vnc-viewer.yml diff --git a/bin/recipes/gnome/notes.md b/bin/recipes/gnome/notes.md new file mode 100644 index 0000000..29ed46b --- /dev/null +++ b/bin/recipes/gnome/notes.md @@ -0,0 +1 @@ +sudo apt-get install task-gnome-desktop \ No newline at end of file diff --git a/bin/recipes/net/net-up b/bin/recipes/net/net-up new file mode 100755 index 0000000..faa413a --- /dev/null +++ b/bin/recipes/net/net-up @@ -0,0 +1,20 @@ +#!/usr/bin/env bash + +sudo mkdir -p /etc/systemd/network +cat <<'EOF' | sudo tee /etc/systemd/network/20-enp0s1.network +[Match] +Name=enp0s1 + +[Network] +DHCP=yes +EOF + +sudo systemctl enable --now systemd-networkd +sudo systemctl enable --now systemd-resolved + +sudo ln -sf /run/systemd/resolve/resolv.conf /etc/resolv.conf + +sudo ip link set enp0s1 up +sudo networkctl reload +sudo networkctl reconfigure enp0s1 + diff --git a/bin/recipes/net/notes.md b/bin/recipes/net/notes.md new file mode 100644 index 0000000..84b1c01 --- /dev/null +++ b/bin/recipes/net/notes.md @@ -0,0 +1,16 @@ +# bring the link up +sudo ip link set enp0s1 up + +# give yourself an address in QEMU's usernet +sudo ip addr add 10.0.2.15/24 dev enp0s1 + +# default gateway for slirp +sudo ip route add default via 10.0.2.2 + +# DNS via slirp (or use 1.1.1.1 if you prefer) +echo "nameserver 10.0.2.3" | sudo tee /etc/resolv.conf > /dev/null + +# sanity checks +ip -4 a show enp0s1 +ping -c2 10.0.2.2 +ping -c2 deb.debian.org diff --git a/bin/recipes/vnc/notes.md b/bin/recipes/vnc/notes.md new file mode 100644 index 0000000..8525f78 --- /dev/null +++ b/bin/recipes/vnc/notes.md @@ -0,0 +1,93 @@ +Here’s what those two pieces typically look like in a QEMU VNC + TLS + SASL setup. + +# `/etc/pki/qemu` (TLS x509 creds) + +QEMU’s `-object tls-creds-x509,...,dir=/etc/pki/qemu,endpoint=server` expects this directory to hold the server’s cert/key and the CA it should trust for client certs. + +Example layout: + +``` +/etc/pki/qemu/ +├── ca-cert.pem # CA cert used to verify client certificates (if verify-peer=yes) +├── server-cert.pem # Server certificate (CN should match the host, or use subjectAltName) +├── server-key.pem # Private key for server-cert.pem (chmod 600, root-only) +└── crl.pem # (optional) Certificate Revocation List +``` + +Typical QEMU arg: + +``` +-object tls-creds-x509,id=tls0,dir=/etc/pki/qemu,endpoint=server,verify-peer=yes +-vnc :0,tls-creds=tls0,sasl +``` + +Notes: + +* `verify-peer=yes` means the VNC client must present a client certificate signed by `ca-cert.pem`. Omit or set `no` if you only want encryption without client certs (and rely on SASL for auth). +* File names above are the conventional ones QEMU looks for in that dir in server mode. +* Permissions matter: keep `server-key.pem` readable only by the QEMU user (e.g., `chmod 600`). + +(If you wanted the client side for certificate auth, you’d create a *client* bundle with `client-cert.pem` / `client-key.pem` and give the server’s CA to the client.) + +# `/etc/sasl2/qemu.conf` (SASL settings) + +This file tells Cyrus SASL how to authenticate for the “qemu” service. A simple, local-password (sasldb) setup might be: + +``` +# /etc/sasl2/qemu.conf +mech_list: scram-sha-256 digest-md5 +pwcheck_method: auxprop +auxprop_plugin: sasldb +sasldb_path: /etc/sasldb2 +``` + +What each line does: + +* `mech_list`: Which SASL mechanisms to allow. (If your build doesn’t have SCRAM, use `digest-md5` and/or `plain`/`login` — but prefer SCRAM or DIGEST over PLAIN.) +* `pwcheck_method: auxprop` + `auxprop_plugin: sasldb`: use the local SASL database. +* `sasldb_path`: where the password DB lives (default is `/etc/sasldb2` on many distros). + +Create users in the sasldb: + +```bash +sudo saslpasswd2 -a qemu -c alice +# (it will prompt for a password) +sudo sasldblistusers2 -f /etc/sasldb2 # verify entry exists +``` + +Then run QEMU with both TLS and SASL: + +```bash +qemu-system-x86_64 \ + -object tls-creds-x509,id=tls0,dir=/etc/pki/qemu,endpoint=server,verify-peer=no \ + -vnc :0,tls-creds=tls0,sasl +``` + +* Client connects over **TLS** (encrypted), then is prompted for **SASL username/password** (“alice” and the password you set). +* If you set `verify-peer=yes`, the client must also present a valid client cert signed by your CA. + +## Quick OpenSSL one-liners (for testing) + +> For production, use a proper CA workflow and strong key handling. + +```bash +# CA +openssl genrsa -out ca.key 4096 +openssl req -x509 -new -nodes -key ca.key -sha256 -days 3650 -out ca-cert.pem -subj "/CN=QEMU Test CA" + +# Server cert +openssl genrsa -out server-key.pem 2048 +openssl req -new -key server-key.pem -out server.csr -subj "/CN=your.host.name" +openssl x509 -req -in server.csr -CA ca-cert.pem -CAkey ca.key -CAcreateserial -out server-cert.pem -days 825 -sha256 +install -m 600 server-key.pem /etc/pki/qemu/ +install -m 644 server-cert.pem ca-cert.pem /etc/pki/qemu/ +``` + +## Common pitfalls + +* **Wrong filenames/paths** in `/etc/pki/qemu` → QEMU won’t find the certs. +* **Permissions too open** on `server-key.pem` → QEMU may refuse or it’s a security risk. +* **SASL mechanism mismatch** → ensure your client supports one from `mech_list`. +* **No TLS but SASL with PLAIN/LOGIN** → credentials go over the wire unencrypted; always pair PLAIN/LOGIN with TLS. + +If you tell me your distro, I can tailor the exact package names (Cyrus SASL modules) and service paths. diff --git a/bin/vm b/bin/vm index 322c850..5923500 100755 --- a/bin/vm +++ b/bin/vm @@ -31,6 +31,11 @@ OptionParser.new do |opt| options.tpm = true end + opt.on('--shell', 'QEMU process must be detached to exec ssh') do + options.shell = true + options.detached = true + end + opt.on('--detached') do options.detached = true end diff --git a/lib/system.rb b/lib/system.rb index bbedf0b..4c56b00 100644 --- a/lib/system.rb +++ b/lib/system.rb @@ -47,4 +47,26 @@ module System def self.arch_to_symbol(arch) normalize_architecture_string(arch) end + + + def self.exec_ssh(port) + require "socket" + + host = "localhost" + cmd = "ssh -p #{port} user@#{host}" + + # Wait until the port is open + puts "Waiting for #{host}:#{port}..." + until begin + TCPSocket.new(host, port).close + true + rescue Errno::ECONNREFUSED, Errno::EHOSTUNREACH, SocketError + false + end + sleep 0.1 + end + + puts "Port open! Executing: #{cmd}" + exec cmd + end end diff --git a/lib/virtual-machine.rb b/lib/virtual-machine.rb index 898c54f..e5b1af4 100644 --- a/lib/virtual-machine.rb +++ b/lib/virtual-machine.rb @@ -109,7 +109,8 @@ module VirtualMachine vars_fd: options[:vars_fd], cpus: [1, System.cpus - 2].max, detach: options[:detached], - tpm: options[:tpm] + tpm: options[:tpm], + shell: options[:shell] ) end diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 60aeb0f..3e86d69 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -13,6 +13,10 @@ module DisplayMode def self.window 2 end + + def self.vnc + 3 + end end module Qemu @@ -81,9 +85,14 @@ module Qemu arch: System::ARCH, cdrom: nil, detach: true, + shell: false, ram: 2048 * 8, cpus: 1, - display: DisplayMode::none + display: DisplayMode::none, + mount: { + wd: nil, + home: nil + } } # for testing only @@ -92,6 +101,7 @@ module Qemu defaults[:display] = DisplayMode.fullscreen defaults[:display] = DisplayMode.window # defaults[:display] = DisplayMode.none + #defaults[:display] = DisplayMode.vnc opts = defaults.merge(options) @@ -119,14 +129,78 @@ module Qemu args += ["-drive", "if=pflash,format=raw,unit=0,readonly=on,file=#{opts[:code_fd]}"] args += ["-drive", "if=pflash,format=raw,unit=1,file=#{opts[:vars_fd]}"] + ssh_port = nil + + if opts[:shell] + ssh_port = rand(4000..9999) + # args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{ssh_port}-:22"] + # args += ['-device', 'virtio-net-device,netdev=net0'] + + # args += ['-netdev', "user,id=n0,hostfwd=tcp:127.0.0.1:#{ssh_port}-10.0.2.15:22"] + # args += ['-device', 'virtio-net,netdev=n0'] + + # args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{ssh_port}-10.0.2.15:22"] + # args += ['-device', 'virtio-net-device,netdev=net0"'] + + args += ['-nic', "user,model=virtio-net-pci,hostfwd=tcp:127.0.0.1:#{ssh_port}-:22"] + + puts "ssh -p #{ssh_port} user@localhost" + + # conf that works + #args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] + end + + # -virtfs local,path=.,mount_tag=hostfs,security_model=passthrough,id=hostfs + # mount -t 9p -o trans=virtio,version=9p2000.L hostfs /mnt + + # hostfs /home 9p trans=virtio,version=9p2000.L,uid=1000,gid=1000,msize=262144,cache=mmap,nofail 0 0 + # hostfs /share 9p trans=virtio,version=9p2000.L,uid=1000,gid=1000,msize=262144,cache=mmap,nofail 0 0 + + if opts[:mount][:wd] + args += ['-virtfs', 'local,path=.,mount_tag=hostfs,security_model=passthrough,id=hostfs'] + end + + if opts[:mount][:home] + #args += ['-homefs', "local,path=#{VMDATA},mount_tag=hostfs,security_model=passthrough,id=hostfs"] + end + if opts[:display] == DisplayMode::none port = 2222 args += ['-nographic'] - args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{port}-:22"] + args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{port}-:22,udp:127.0.0.1:6544-:6544=on"] #args += ['-device', 'e1000,netdev=net0'] args += ['-device', 'virtio-net-pci,netdev=net0'] puts "ssh -p #{port} user@localhost" + + + elsif opts[:display] == DisplayMode::vnc + # Note: this outputs serial on the console + #args += ['-nographic'] + + args += ["-display", "none"] + args += ["-device", "virtio-gpu-pci"] + args += ["-device", "virtio-keyboard-pci"] + args += ["-device", "virtio-mouse-pci"] + + args += ['-vnc', '127.0.0.1:0,password=on'] + + # tunnel ssh -L 5900:127.0.0.1:5900 user@your-host + # -monitor unix:/tmp/qemu-mon,server,nowait + # -vnc 127.0.0.1:0,password=on + # printf 'change vnc password\nMySecret\n' | socat - UNIX-CONNECT:/tmp/qemu-mon + + + # SASL auth (username + password) + # -vnc :0,sasl + + # TLS (certificates, optional password) + # -object tls-creds-x509,id=tls0,... + # -vnc :0,tls-creds=tls0 + + # qemu-system-x86_64 \ + # -object tls-creds-x509,id=tls0,dir=/etc/pki/qemu,endpoint=server \ + # -vnc :0,tls-creds=tls0,sasl else #args += ["-device", "virtio-gpu-device"] if opts[:display] == DisplayMode::fullscreen @@ -156,7 +230,18 @@ module Qemu args += ["-device", "virtio-gpu"] #args += ['-nic', 'user,model=virtio-net-pci'] - args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] + unless opts[:shell] + args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] + end + + + ### TODO: remove + port = 2222 + args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{port}-:22,hostfwd=udp:127.0.0.1:6544-:6544"] + args += ['-virtfs', 'local,path=.,mount_tag=hostfs,security_model=passthrough,id=hostfs'] + ### TODO END + + # macOS vmnet (shares Mac’s LAN) # -netdev vmnet-shared,id=n1 \ # -device virtio-net-pci,netdev=n1 @@ -211,6 +296,8 @@ module Qemu # -netdev vmnet-shared,id=net0 end + args += ['-monitor', 'stdio'] + # args += ["-device", "virtio-net,netdev=n0", "-netdev", "user,id=n0"] # user-mode NAT # optional: uncomment to run headless with VNC on :5901 # args += ["-display", "none", "-vnc", "127.0.0.1:1"] @@ -224,6 +311,11 @@ module Qemu pid = Process.spawn(*cmd, pgroup: false, out: log, err: log) Process.detach(pid) puts "QEMU pid=#{pid}" + + if opts[:shell] + System.exec_ssh(ssh_port) + end + else pid = Process.spawn(*cmd, pgroup: true, out: $stdout, err: $stderr) Process.wait(pid) diff --git a/recipes/vnc-viewer.yml b/recipes/vnc-viewer.yml new file mode 100644 index 0000000..8a755fb --- /dev/null +++ b/recipes/vnc-viewer.yml @@ -0,0 +1,45 @@ + # # 1) Build & install FLTK 1.4 to /opt/fltk-1.4 + # git clone https://github.com/fltk/fltk.git + # cd fltk + # git checkout branch-1.4 + # cmake -S . -B build \ + # -DOPTION_BUILD_SHARED_LIBS=ON \ + # -DOPTION_USE_SYSTEM_LIBPNG=ON \ + # -DOPTION_USE_SYSTEM_LIBJPEG=ON \ + # -DOPTION_USE_SYSTEM_ZLIB=ON \ + # -DCMAKE_INSTALL_PREFIX=/opt/fltk-1.4 + # cmake --build build -j + # sudo cmake --install build + + # # 2) Point pkg-config and CMake at the new install + # export PKG_CONFIG_PATH=/opt/fltk-1.4/lib/pkgconfig:$PKG_CONFIG_PATH + # export CMAKE_PREFIX_PATH=/opt/fltk-1.4:$CMAKE_PREFIX_PATH + # # (optional but sometimes necessary) + # export FLTK_DIR=/opt/fltk-1.4/lib/cmake/FLTK + + +packages: + - cmake + - ninja + - pkg-config + - fltk + - jpeg-turbo + - libpng + - zlib + - gnutls + - ffmpeg + +# --recursive +repository: + url: https://github.com/TigerVNC/tigervnc.git + url: v1.15.0 + +steps: + - mkdir build && cd build + - | + cmake -G Ninja .. \ + -DCMAKE_BUILD_TYPE=Release \ + -DBUILD_VIEWER=ON \ + -DBUILD_SERVER=OFF \ + -DWITH_GNUTLS=ON + - ninja \ No newline at end of file From 7b35ea96bb34d6155f012295b4c94ff946ddefd5 Mon Sep 17 00:00:00 2001 From: Artur Gurgul Date: Fri, 29 Aug 2025 15:52:53 +0200 Subject: [PATCH 51/52] save --- bin/sq | 100 +++++++++++++++++++++++++++++++++++++++++++++++++ lib/vm/qemu.rb | 16 ++++++++ 2 files changed, 116 insertions(+) create mode 100755 bin/sq diff --git a/bin/sq b/bin/sq new file mode 100755 index 0000000..78b9236 --- /dev/null +++ b/bin/sq @@ -0,0 +1,100 @@ +#!/usr/bin/env ruby +require 'socket' + +# Path to QEMU monitor socket +sock_path = "/tmp/qemu-monitor.sock" + +# Connect to UNIX socket +sock = UNIXSocket.new(sock_path) + +# Read greeting from QEMU (optional) +puts sock.readline rescue nil + +# Function to send a single keystroke +def send_key(sock, key) + cmd = "sendkey #{key}\n" + puts ">>> #{cmd.strip}" + sock.write(cmd) +end + +def char_to_key(ch) + case ch + # Control characters + when "\n" then "ret" + when "\t" then "tab" + when " " then "spc" + when "\e" then "esc" + when "\b" then "backspace" + + # Lowercase letters + when "a".."z" then ch + + # Uppercase letters (shift + letter) + when "A".."Z" then "shift-#{ch.downcase}" + + # Digits + when "0".."9" then ch + + # Shifted number row + when "!" then "shift-1" + when "@" then "shift-2" + when "#" then "shift-3" + when "$" then "shift-4" + when "%" then "shift-5" + when "^" then "shift-6" + when "&" then "shift-7" + when "*" then "shift-8" + when "(" then "shift-9" + when ")" then "shift-0" + + when "-" then "minus" + when "_" then "shift-minus" + when "=" then "equal" + when "+" then "shift-equal" + when "[" then "bracket_left" + when "{" then "shift-bracket_left" + when "]" then "bracket_right" + when "}" then "shift-bracket_right" + when "\\" then "backslash" + when "|" then "shift-backslash" + when ";" then "semicolon" + when ":" then "shift-semicolon" + when "'" then "apostrophe" + when "\"" then "shift-apostrophe" + when "," then "comma" + when "<" then "shift-comma" + when "." then "dot" + when ">" then "shift-dot" + when "/" then "slash" + when "?" then "shift-slash" + when "`" then "grave_accent" + when "~" then "shift-grave_accent" + + else + raise "Unsupported character: #{ch.inspect}" + end +end + +if ARGV[0] + puts ARGV[0] + input = ARGV[0] + puts "Typing: #{input.inspect}" + + input.chars.each do |ch| + send_key(sock, char_to_key(ch)) + sleep 0.1 + end + +else + # Example: type "hello" + %w[p a s s ret].each do |k| + send_key(sock, k) + sleep 0.1 + end +end + +sock.close + + +# (qemu) screendump /Volumes/Cache/homes/debian/guaset.ppm +# ffmpeg -y -i guaset.ppm guest.png \ No newline at end of file diff --git a/lib/vm/qemu.rb b/lib/vm/qemu.rb index 3e86d69..506e6bc 100644 --- a/lib/vm/qemu.rb +++ b/lib/vm/qemu.rb @@ -152,6 +152,7 @@ module Qemu # -virtfs local,path=.,mount_tag=hostfs,security_model=passthrough,id=hostfs # mount -t 9p -o trans=virtio,version=9p2000.L hostfs /mnt + # sudo mount -t 9p hostfs /home/user/Share -o trans=virtio,version=9p2000.L,uid=1000,gid=1000,msize=262144,cache=mmap # hostfs /home 9p trans=virtio,version=9p2000.L,uid=1000,gid=1000,msize=262144,cache=mmap,nofail 0 0 # hostfs /share 9p trans=virtio,version=9p2000.L,uid=1000,gid=1000,msize=262144,cache=mmap,nofail 0 0 @@ -237,6 +238,7 @@ module Qemu ### TODO: remove port = 2222 + args += ['-device', 'virtio-net-pci,netdev=net0'] args += ['-netdev', "user,id=net0,hostfwd=tcp:127.0.0.1:#{port}-:22,hostfwd=udp:127.0.0.1:6544-:6544"] args += ['-virtfs', 'local,path=.,mount_tag=hostfs,security_model=passthrough,id=hostfs'] ### TODO END @@ -326,3 +328,17 @@ module Qemu end + + +## Works on MacOS= +# -monitor unix:/tmp/qemu-monitor.sock,server,nowait +# nc -U /tmp/qemu-monitor.sock +# instead of args += ['-monitor', 'stdio'] + + + + +# 9P +# sudo mount -t 9p hostfs /home/user/Share \ +# -o trans=virtio,version=9p2000.L,msize=262144,cache=mmap,access=any,dfltuid=1000,dfltgid=1000 +# sudo chown -hR 1000:1000 /home/user/Share \ No newline at end of file From 1c65089361e07b349b3ccc96e3b0eb2054dfabc1 Mon Sep 17 00:00:00 2001 From: Emperor Date: Sun, 7 Sep 2025 01:03:14 -0500 Subject: [PATCH 52/52] Save --- recipes/nginx/debian.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/recipes/nginx/debian.yml b/recipes/nginx/debian.yml index 3d19034..48d2245 100644 --- a/recipes/nginx/debian.yml +++ b/recipes/nginx/debian.yml @@ -1,8 +1,7 @@ packages: - build-essential - - libpcre3 - - libpcre3-dev + - libpcre2-dev - zlib1g - zlib1g-dev - libssl-dev