commit 08fa407f5769310d94188e8ad774607a45a936a1 Author: codex Date: Fri Jun 5 07:00:43 2026 +0000 Initial commit: Consolidated DevOps scripts and K8s manifests diff --git a/k8s/coming-soon-ingress.yaml b/k8s/coming-soon-ingress.yaml new file mode 100644 index 0000000..c48d85a --- /dev/null +++ b/k8s/coming-soon-ingress.yaml @@ -0,0 +1,70 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: coming-soon-fallback + namespace: default + annotations: + traefik.ingress.kubernetes.io/router.priority: "1" +spec: + ingressClassName: traefik + rules: + - host: "*.thefoldwithin.earth" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 + - host: "*.intellecton.one" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 + - host: "*.atlanta.pentagon" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 + - host: "thefoldwithin.earth" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 + - host: "intellecton.one" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 + - host: "atlanta.pentagon" + http: + paths: + - path: / + pathType: Prefix + backend: + service: + name: coming-soon-svc + port: + number: 80 diff --git a/k8s/makeanyplace-org.yaml b/k8s/makeanyplace-org.yaml new file mode 100644 index 0000000..f453264 --- /dev/null +++ b/k8s/makeanyplace-org.yaml @@ -0,0 +1,68 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: makeanyplace-org + namespace: default + labels: + app: makeanyplace-org +spec: + replicas: 2 + selector: + matchLabels: + app: makeanyplace-org + template: + metadata: + labels: + app: makeanyplace-org + spec: + containers: + - name: org + image: remember.thefoldwithin.earth/mrhavens/makeanyplace-org:latest + imagePullPolicy: Always + ports: + - containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: makeanyplace-org-svc + namespace: default +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: makeanyplace-org + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: makeanyplace-org-ingress + namespace: default + annotations: + kubernetes.io/ingress.class: traefik +spec: + rules: + - host: makeanyplace.org + http: + paths: + - backend: + service: + name: makeanyplace-org-svc + port: + number: 80 + path: / + pathType: Prefix + - host: www.makeanyplace.org + http: + paths: + - backend: + service: + name: makeanyplace-org-svc + port: + number: 80 + path: / + pathType: Prefix diff --git a/k8s/makeanyplace-staging.yaml b/k8s/makeanyplace-staging.yaml new file mode 100644 index 0000000..e61c6cd --- /dev/null +++ b/k8s/makeanyplace-staging.yaml @@ -0,0 +1,56 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: makeanyplace-web-staging + namespace: default + labels: + app: makeanyplace-web-staging +spec: + replicas: 1 + selector: + matchLabels: + app: makeanyplace-web-staging + template: + metadata: + labels: + app: makeanyplace-web-staging + spec: + containers: + - image: remember.thefoldwithin.earth/mrhavens/makeanyplace-web:staging + imagePullPolicy: Always + name: web + ports: + - containerPort: 80 + protocol: TCP +--- +apiVersion: v1 +kind: Service +metadata: + name: makeanyplace-web-staging-svc + namespace: default +spec: + ports: + - port: 80 + protocol: TCP + targetPort: 80 + selector: + app: makeanyplace-web-staging + type: ClusterIP +--- +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + name: makeanyplace-staging-ingress + namespace: default +spec: + rules: + - host: makeanyplace.staging + http: + paths: + - backend: + service: + name: makeanyplace-web-staging-svc + port: + number: 80 + path: / + pathType: Prefix diff --git a/k8s/parking.yaml b/k8s/parking.yaml new file mode 100644 index 0000000..273050e --- /dev/null +++ b/k8s/parking.yaml @@ -0,0 +1,149 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: makeanyplace-parking-html + namespace: default +data: + index.html: | + + + + + + MakeAnyplace - The Sovereign Maker Network + + + +
+
+

MakeAnyplace.

+

The sovereign maker network is currently undergoing deep architectural upgrades. The monastery doors will open soon.

+
+ Building the future +
+
+ + +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: makeanyplace-parking + namespace: default +spec: + replicas: 2 + selector: + matchLabels: + app: makeanyplace-web + template: + metadata: + labels: + app: makeanyplace-web + variant: parking + spec: + containers: + - name: nginx + image: nginx:alpine + ports: + - containerPort: 80 + volumeMounts: + - name: html-volume + mountPath: /usr/share/nginx/html + volumes: + - name: html-volume + configMap: + name: makeanyplace-parking-html diff --git a/k8s/redis.yaml b/k8s/redis.yaml new file mode 100644 index 0000000..8623c5f --- /dev/null +++ b/k8s/redis.yaml @@ -0,0 +1,32 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: redis + labels: + app: redis +spec: + replicas: 1 + selector: + matchLabels: + app: redis + template: + metadata: + labels: + app: redis + spec: + containers: + - name: redis + image: redis:alpine + ports: + - containerPort: 6379 +--- +apiVersion: v1 +kind: Service +metadata: + name: redis +spec: + selector: + app: redis + ports: + - port: 6379 + targetPort: 6379 diff --git a/k8s/update_ingress.yaml b/k8s/update_ingress.yaml new file mode 100644 index 0000000..1f418f1 --- /dev/null +++ b/k8s/update_ingress.yaml @@ -0,0 +1,43 @@ +apiVersion: networking.k8s.io/v1 +kind: Ingress +metadata: + annotations: + kubernetes.io/ingress.class: traefik + name: makeanyplace-web-ingress + namespace: default +spec: + rules: + - host: makeanyplace.com + http: + paths: + - backend: + service: + name: makeanyplace-api + port: + number: 8000 + path: /api + pathType: Prefix + - backend: + service: + name: makeanyplace-web-svc + port: + number: 80 + path: / + pathType: Prefix + - host: www.makeanyplace.com + http: + paths: + - backend: + service: + name: makeanyplace-api + port: + number: 8000 + path: /api + pathType: Prefix + - backend: + service: + name: makeanyplace-web-svc + port: + number: 80 + path: / + pathType: Prefix diff --git a/scripts/deploy/deploy_makeanyplace.py b/scripts/deploy/deploy_makeanyplace.py new file mode 100644 index 0000000..a4a4d0c --- /dev/null +++ b/scripts/deploy/deploy_makeanyplace.py @@ -0,0 +1,46 @@ +import urllib.request, json, sys + +cf_token = "cfat_ub1jXkUJinoJDmhiUJJpwQni2rSVnLnysHzvcwwH351fdff8" +cf_account = "e3584bc80d5c6df89d93078172898d73" +pb_key = "pk1_6e51c25cc1f0ded78246f0a4cd4a3cef404c47308dcc2f8b66bf8065ec6350bf" +pb_secret = "sk1_7dc521cdfb1a987285af2b811333e2fed77ba1013559c6671e25f948305d4f29" + +domains = ["makeanyplace.com", "makeanyplace.org"] + +for domain in domains: + print(f"--- Routing {domain} ---") + + # 1. Create Zone in Cloudflare + cf_req = urllib.request.Request( + 'https://api.cloudflare.com/client/v4/zones', + data=json.dumps({"name": domain, "account": {"id": cf_account}, "type": "full"}).encode(), + headers={'Authorization': f'Bearer {cf_token}', 'Content-Type': 'application/json'} + ) + + try: + cf_resp = json.loads(urllib.request.urlopen(cf_req).read()) + nameservers = cf_resp['result']['name_servers'] + print(f"Cloudflare Zone Created for {domain}. Nameservers: {nameservers}") + except urllib.error.HTTPError as e: + err_body = e.read().decode() + if "already exists" in err_body or e.code == 400: + print(f"Zone {domain} already exists in Cloudflare. Fetching...") + cf_get = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones?name={domain}', + headers={'Authorization': f'Bearer {cf_token}', 'Content-Type': 'application/json'} + ) + cf_resp = json.loads(urllib.request.urlopen(cf_get).read()) + nameservers = cf_resp['result'][0]['name_servers'] + print(f"Fetched existing zone nameservers for {domain}: {nameservers}") + else: + print(f"CF Error for {domain}:", err_body) + continue + + # 2. Update Porkbun Nameservers + pb_req = urllib.request.Request( + f'https://api.porkbun.com/api/json/v3/domain/updateNs/{domain}', + data=json.dumps({"apikey": pb_key, "secretapikey": pb_secret, "ns": nameservers}).encode(), + headers={'Content-Type': 'application/json'} + ) + pb_resp = json.loads(urllib.request.urlopen(pb_req).read()) + print(f"Porkbun Update Response for {domain}: {pb_resp}\n") diff --git a/scripts/deploy/run_local_bots.sh b/scripts/deploy/run_local_bots.sh new file mode 100755 index 0000000..8e5549b --- /dev/null +++ b/scripts/deploy/run_local_bots.sh @@ -0,0 +1,9 @@ +#!/bin/bash +cd /home/antigravity/scratch/makeanyplace-backend +source venv/bin/activate +export DB_HOST=postgres.local +export REDIS_HOST=127.0.0.1 +# Run the aggregator +python3 event_aggregator.py +# Copy the updated events file to the local development site frontend +cp live_events.json ../makeanyplace-web/src/live_events.json diff --git a/scripts/dns/check_cf_rules.py b/scripts/dns/check_cf_rules.py new file mode 100644 index 0000000..15109d9 --- /dev/null +++ b/scripts/dns/check_cf_rules.py @@ -0,0 +1,27 @@ +import urllib.request +import json + +token = "LfObAyuwjRs_X7Emp-kNl4AFjU1FX0XdLqVwgX0p" +zone_id = "f019fee4c5b58c48e4493b931f800ff5" # makeanyplace.com + +print("--- Page Rules ---") +req = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones/{zone_id}/pagerules', + headers={'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} +) +try: + resp = json.loads(urllib.request.urlopen(req).read()) + print(json.dumps(resp, indent=2)) +except Exception as e: + print("Error:", e) + +print("\n--- Rulesets ---") +req2 = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones/{zone_id}/rulesets', + headers={'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} +) +try: + resp2 = json.loads(urllib.request.urlopen(req2).read()) + print(json.dumps(resp2, indent=2)) +except Exception as e: + print("Error:", e) diff --git a/scripts/dns/check_cloudflare.py b/scripts/dns/check_cloudflare.py new file mode 100644 index 0000000..969afaf --- /dev/null +++ b/scripts/dns/check_cloudflare.py @@ -0,0 +1,48 @@ +import urllib.request +import json +import sys + +# Try both tokens +tokens = [ + "LfObAyuwjRs_X7Emp-kNl4AFjU1FX0XdLqVwgX0p", + "cfat_ub1jXkUJinoJDmhiUJJpwQni2rSVnLnysHzvcwwH351fdff8" +] + +domains = ["makeanyplace.com", "makeanyplace.org"] + +for token in tokens: + print(f"=== Trying Token: {token[:8]}... ===") + for domain in domains: + # Get Zone ID + req = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones?name={domain}', + headers={'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} + ) + try: + resp = json.loads(urllib.request.urlopen(req).read()) + if not resp.get('result'): + print(f"Zone {domain} not found with this token.") + continue + + zone = resp['result'][0] + zone_id = zone['id'] + status = zone['status'] + print(f"Domain: {domain}, Zone ID: {zone_id}, Status: {status}") + print(f"Nameservers: {zone.get('name_servers', [])}") + + # Fetch DNS Records + rec_req = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones/{zone_id}/dns_records', + headers={'Authorization': f'Bearer {token}', 'Content-Type': 'application/json'} + ) + rec_resp = json.loads(urllib.request.urlopen(rec_req).read()) + print("DNS Records at Cloudflare:") + for rec in rec_resp.get('result', []): + print(f" Type: {rec.get('type')}, Name: {rec.get('name')}, Content: {rec.get('content')}, Proxied: {rec.get('proxied')}") + + except urllib.error.HTTPError as he: + print(f"HTTP Error for {domain}: {he.code} {he.reason}") + print(he.read().decode()) + except Exception as e: + print(f"Error for {domain}: {e}") + print() diff --git a/scripts/dns/check_porkbun.py b/scripts/dns/check_porkbun.py new file mode 100644 index 0000000..6aad231 --- /dev/null +++ b/scripts/dns/check_porkbun.py @@ -0,0 +1,37 @@ +import urllib.request +import json + +pb_key = "pk1_6e51c25cc1f0ded78246f0a4cd4a3cef404c47308dcc2f8b66bf8065ec6350bf" +pb_secret = "sk1_7dc521cdfb1a987285af2b811333e2fed77ba1013559c6671e25f948305d4f29" + +domains = ["makeanyplace.com", "makeanyplace.org"] + +for domain in domains: + print(f"=== Domain: {domain} ===") + + # Get Nameservers + ns_req = urllib.request.Request( + f'https://api.porkbun.com/api/json/v3/domain/getNs/{domain}', + data=json.dumps({"apikey": pb_key, "secretapikey": pb_secret}).encode(), + headers={'Content-Type': 'application/json'} + ) + try: + ns_resp = json.loads(urllib.request.urlopen(ns_req).read()) + print(f"Nameservers: {ns_resp.get('ns', [])}") + except Exception as e: + print(f"Error getting NS: {e}") + + # Get DNS Records + dns_req = urllib.request.Request( + f'https://api.porkbun.com/api/json/v3/domain/dns/retrieve/{domain}', + data=json.dumps({"apikey": pb_key, "secretapikey": pb_secret}).encode(), + headers={'Content-Type': 'application/json'} + ) + try: + dns_resp = json.loads(urllib.request.urlopen(dns_req).read()) + print("DNS Records:") + for rec in dns_resp.get('records', []): + print(f" Type: {rec.get('type')}, Name: {rec.get('name')}, Content: {rec.get('content')}") + except Exception as e: + print(f"Error getting DNS: {e}") + print() diff --git a/scripts/dns/update_dns.py b/scripts/dns/update_dns.py new file mode 100644 index 0000000..d9322ea --- /dev/null +++ b/scripts/dns/update_dns.py @@ -0,0 +1,41 @@ +import urllib.request, json, sys + +cf_token = "LfObAyuwjRs_X7Emp-kNl4AFjU1FX0XdLqVwgX0p" +cf_account = "e3584bc80d5c6df89d93078172898d73" +pb_key = "pk1_6e51c25cc1f0ded78246f0a4cd4a3cef404c47308dcc2f8b66bf8065ec6350bf" +pb_secret = "sk1_7dc521cdfb1a987285af2b811333e2fed77ba1013559c6671e25f948305d4f29" +domain = "mrhavens.one" + +# 1. Create Zone in Cloudflare +cf_req = urllib.request.Request( + 'https://api.cloudflare.com/client/v4/zones', + data=json.dumps({"name": domain, "account": {"id": cf_account}, "type": "full"}).encode(), + headers={'Authorization': f'Bearer {cf_token}', 'Content-Type': 'application/json'} +) +try: + cf_resp = json.loads(urllib.request.urlopen(cf_req).read()) + nameservers = cf_resp['result']['name_servers'] + print("Cloudflare Zone Created. Nameservers:", nameservers) +except urllib.error.HTTPError as e: + err_body = e.read().decode() + if "already exists" in err_body or e.code == 400: + print("Zone already exists. Fetching...") + cf_get = urllib.request.Request( + f'https://api.cloudflare.com/client/v4/zones?name={domain}', + headers={'Authorization': f'Bearer {cf_token}', 'Content-Type': 'application/json'} + ) + cf_resp = json.loads(urllib.request.urlopen(cf_get).read()) + nameservers = cf_resp['result'][0]['name_servers'] + print("Fetched existing zone nameservers:", nameservers) + else: + print("CF Error:", err_body) + sys.exit(1) + +# 2. Update Porkbun Nameservers +pb_req = urllib.request.Request( + f'https://api.porkbun.com/api/json/v3/domain/updateNs/{domain}', + data=json.dumps({"apikey": pb_key, "secretapikey": pb_secret, "ns": nameservers}).encode(), + headers={'Content-Type': 'application/json'} +) +pb_resp = json.loads(urllib.request.urlopen(pb_req).read()) +print("Porkbun Update Response:", pb_resp) diff --git a/scripts/dns/update_porkbun_ns.py b/scripts/dns/update_porkbun_ns.py new file mode 100644 index 0000000..2701642 --- /dev/null +++ b/scripts/dns/update_porkbun_ns.py @@ -0,0 +1,20 @@ +import urllib.request, json + +pb_key = "pk1_6e51c25cc1f0ded78246f0a4cd4a3cef404c47308dcc2f8b66bf8065ec6350bf" +pb_secret = "sk1_7dc521cdfb1a987285af2b811333e2fed77ba1013559c6671e25f948305d4f29" +domain = "makeanyplace.org" +nameservers = ["adrian.ns.cloudflare.com", "buck.ns.cloudflare.com"] + +print(f"Updating nameservers for {domain} on Porkbun...") + +pb_req = urllib.request.Request( + f'https://api.porkbun.com/api/json/v3/domain/updateNs/{domain}', + data=json.dumps({"apikey": pb_key, "secretapikey": pb_secret, "ns": nameservers}).encode(), + headers={'Content-Type': 'application/json'} +) + +try: + pb_resp = json.loads(urllib.request.urlopen(pb_req).read()) + print(f"Success! Porkbun Response: {pb_resp}") +except Exception as e: + print(f"Failed to update Porkbun: {e}") diff --git a/scripts/sync/sync_gitlab.py b/scripts/sync/sync_gitlab.py new file mode 100644 index 0000000..ae24365 --- /dev/null +++ b/scripts/sync/sync_gitlab.py @@ -0,0 +1,117 @@ +import os +import subprocess +import json +import urllib.request +import urllib.error +import base64 +import shutil +import time + +GITLAB_API = "https://gitlab.com/api/v4" +GITLAB_TOKEN = "glpat-KrN8svq4D9QeQD6-_p_r7GM6MQpvOjEKdTozN3R4cg8.01.171o6kz82" +FORGEJO_LOCAL_URL = "http://forgejo.local/api/v1" +AUTH = ("mrhavens", "Aok4y2k!") +BACKUP_DIR = "/home/antigravity/scratch/gitlab_backups" + +os.makedirs(BACKUP_DIR, exist_ok=True) + +def _auth_headers(): + auth_str = f"{AUTH[0]}:{AUTH[1]}" + b64_auth = base64.b64encode(auth_str.encode('ascii')).decode('ascii') + return { + "Authorization": f"Basic {b64_auth}", + "Content-Type": "application/json", + "User-Agent": "curl/7.81.0" + } + +def get_forgejo_repos(): + repos = [] + page = 1 + while True: + req = urllib.request.Request(f"{FORGEJO_LOCAL_URL}/user/repos?limit=50&page={page}", headers=_auth_headers()) + try: + with urllib.request.urlopen(req) as resp: + if resp.status != 200: + break + data = json.loads(resp.read().decode('utf-8')) + if not data: + break + repos.extend(data) + page += 1 + except Exception as e: + break + return {r['name'] for r in repos} + +def get_gitlab_repos(): + repos = [] + page = 1 + while True: + req = urllib.request.Request(f"{GITLAB_API}/projects?owned=true&simple=true&per_page=100&page={page}") + req.add_header("PRIVATE-TOKEN", GITLAB_TOKEN) + try: + with urllib.request.urlopen(req) as resp: + data = json.loads(resp.read().decode('utf-8')) + if not data: + break + repos.extend(data) + page += 1 + except Exception as e: + print(f"Failed to fetch GitLab repos: {e}") + break + return repos + +print("Fetching existing repos from forgejo.local...") +local_repo_names = get_forgejo_repos() +print(f"Found {len(local_repo_names)} repos on forgejo.local.") + +print("Fetching repos from GitLab cautiously...") +gitlab_repos = get_gitlab_repos() +print(f"Found {len(gitlab_repos)} repos on GitLab.") + +for repo in gitlab_repos: + name = repo['path'] + clone_url = repo['http_url_to_repo'].replace("https://", f"https://oauth2:{GITLAB_TOKEN}@") + + print(f"\n--- Processing GitLab Repo: {name} ---") + + # Sleep to avoid triggering rate limits / security + time.sleep(10) + + # 1. Create on forgejo.local if it doesn't exist + if name not in local_repo_names: + print(f"Creating {name} on forgejo.local...") + req = urllib.request.Request(f"{FORGEJO_LOCAL_URL}/user/repos", data=json.dumps({"name": name, "private": repo.get('visibility') == 'private'}).encode('utf-8'), headers=_auth_headers(), method="POST") + try: + urllib.request.urlopen(req) + local_repo_names.add(name) + except urllib.error.HTTPError as e: + if e.code != 409: + print(f"Failed to create repo {name}: {e.code} {e.reason}") + continue + else: + print(f"{name} already exists on forgejo.local.") + + # 2. Clone and Push + repo_dir = os.path.join(BACKUP_DIR, name) + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + + print(f"Cloning {name} from GitLab...") + try: + subprocess.run(["git", "clone", "--mirror", clone_url, repo_dir], check=True, capture_output=True) + + print(f"Pushing {name} to forgejo.local...") + push_url = f"http://{AUTH[0]}:{AUTH[1]}@forgejo.local/mrhavens/{name}.git" + subprocess.run(["git", "push", "--mirror", push_url], cwd=repo_dir, check=True, capture_output=True) + print(f"Successfully mirrored {name}!") + + except subprocess.CalledProcessError as e: + print(f"Error processing {name}: {e}") + if e.stderr: + print(e.stderr.decode('utf-8', errors='ignore')) + + # Clean up to save space + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + +print("\nGitLab synchronization complete.") diff --git a/scripts/sync/sync_repos.py b/scripts/sync/sync_repos.py new file mode 100644 index 0000000..f128983 --- /dev/null +++ b/scripts/sync/sync_repos.py @@ -0,0 +1,63 @@ +import os +import subprocess +import urllib.request +import json +import base64 +import tempfile + +GITLAB_TOKEN = "glpat-KrN8svq4D9QeQD6-_p_r7GM6MQpvOjEKdTozN3R4cg8.01.171o6kz82" +FORGEJO_AUTH = "mrhavens:Aok4y2k!" + +def get_gitlab_repos(): + req = urllib.request.Request("https://gitlab.com/api/v4/users/mrhavens/projects?per_page=100") + req.add_header("PRIVATE-TOKEN", GITLAB_TOKEN) + with urllib.request.urlopen(req) as f: + return [r for r in json.loads(f.read().decode('utf-8'))] + +def get_forgejo_repos(): + req = urllib.request.Request("https://remember.thefoldwithin.earth/api/v1/user/repos?limit=100") + req.add_header("User-Agent", "Mozilla/5.0") + auth = base64.b64encode(FORGEJO_AUTH.encode("ascii")).decode("ascii") + req.add_header("Authorization", f"Basic {auth}") + with urllib.request.urlopen(req) as f: + return [r["name"] for r in json.loads(f.read().decode('utf-8'))] + +def create_forgejo_repo(name, description): + req = urllib.request.Request("https://remember.thefoldwithin.earth/api/v1/user/repos", method="POST") + req.add_header("User-Agent", "Mozilla/5.0") + auth = base64.b64encode(FORGEJO_AUTH.encode("ascii")).decode("ascii") + req.add_header("Authorization", f"Basic {auth}") + req.add_header("Content-Type", "application/json") + data = json.dumps({"name": name, "description": description or ""}).encode("utf-8") + try: + with urllib.request.urlopen(req, data=data) as f: + pass + except urllib.error.HTTPError as e: + if e.code == 409: + print(f"{name} already exists on Forgejo") + else: + print(f"Failed to create {name} on Forgejo: {e}") + +gl_repos = get_gitlab_repos() +fj_repos_names = set(get_forgejo_repos()) + +missing = [r for r in gl_repos if r["name"] not in fj_repos_names] +print(f"Found {len(missing)} missing repositories.") + +for repo in missing: + name = repo["name"] + desc = repo.get("description", "") + print(f"Processing {name}...") + create_forgejo_repo(name, desc) + + # Clone and push + gl_url = f"https://oauth2:{GITLAB_TOKEN}@gitlab.com/mrhavens/{name}.git" + fj_url = f"https://mrhavens:Aok4y2k!@remember.thefoldwithin.earth/mrhavens/{name}.git" + + try: + with tempfile.TemporaryDirectory() as tmpdir: + subprocess.run(["git", "clone", "--bare", gl_url, tmpdir], check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + subprocess.run(["git", "push", "--mirror", fj_url], cwd=tmpdir, check=True, stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) + print(f"Successfully mirrored {name}") + except subprocess.CalledProcessError as e: + print(f"Git operation failed for {name}: {e}") diff --git a/scripts/sync/sync_repos_from_remember.py b/scripts/sync/sync_repos_from_remember.py new file mode 100644 index 0000000..b0680eb --- /dev/null +++ b/scripts/sync/sync_repos_from_remember.py @@ -0,0 +1,96 @@ +import os +import subprocess +import json +import urllib.request +import urllib.error +import base64 +import shutil + +REMEMBER_URL = "https://remember.thefoldwithin.earth/api/v1" +FORGEJO_LOCAL_URL = "http://forgejo.local/api/v1" +AUTH = ("mrhavens", "Aok4y2k!") +BACKUP_DIR = "/home/antigravity/scratch/remember_backups" + +os.makedirs(BACKUP_DIR, exist_ok=True) + +def _auth_headers(): + auth_str = f"{AUTH[0]}:{AUTH[1]}" + b64_auth = base64.b64encode(auth_str.encode('ascii')).decode('ascii') + return { + "Authorization": f"Basic {b64_auth}", + "Content-Type": "application/json", + "User-Agent": "curl/7.81.0" + } + +def get_all_repos(api_url): + repos = [] + page = 1 + while True: + req = urllib.request.Request(f"{api_url}/user/repos?limit=50&page={page}", headers=_auth_headers()) + try: + with urllib.request.urlopen(req) as resp: + if resp.status != 200: + print(f"Error fetching repos: {resp.status}") + break + data = json.loads(resp.read().decode('utf-8')) + if not data: + break + repos.extend(data) + page += 1 + except Exception as e: + print(f"Failed to fetch {api_url}: {e}") + break + return repos + +print("Fetching repos from remember...") +remember_repos = get_all_repos(REMEMBER_URL) +print(f"Found {len(remember_repos)} repos on remember.thefoldwithin.earth") + +print("Fetching repos from forgejo.local...") +local_repos = get_all_repos(FORGEJO_LOCAL_URL) +local_repo_names = {r['name'] for r in local_repos} +print(f"Found {len(local_repos)} repos on forgejo.local") + +for repo in remember_repos: + name = repo['name'] + clone_url = repo['clone_url'].replace("https://", f"https://{AUTH[0]}:{AUTH[1]}@") + + print(f"\n--- Processing {name} ---") + + # 1. Create on forgejo.local if it doesn't exist + if name not in local_repo_names: + print(f"Creating {name} on forgejo.local...") + req = urllib.request.Request(f"{FORGEJO_LOCAL_URL}/user/repos", data=json.dumps({"name": name, "private": repo['private']}).encode('utf-8'), headers=_auth_headers(), method="POST") + try: + urllib.request.urlopen(req) + except urllib.error.HTTPError as e: + if e.code != 409: + print(f"Failed to create repo {name}: {e.code} {e.reason}") + continue + else: + print(f"{name} already exists on forgejo.local.") + + # 2. Clone and Push + repo_dir = os.path.join(BACKUP_DIR, name) + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + + print(f"Cloning {name} from remember...") + try: + subprocess.run(["git", "clone", "--mirror", clone_url, repo_dir], check=True, capture_output=True) + + print(f"Pushing {name} to forgejo.local...") + push_url = f"http://{AUTH[0]}:{AUTH[1]}@forgejo.local/mrhavens/{name}.git" + subprocess.run(["git", "push", "--mirror", push_url], cwd=repo_dir, check=True, capture_output=True) + print(f"Successfully mirrored {name}!") + + except subprocess.CalledProcessError as e: + print(f"Error processing {name}: {e}") + if e.stderr: + print(e.stderr.decode('utf-8', errors='ignore')) + + # Clean up to save space + if os.path.exists(repo_dir): + shutil.rmtree(repo_dir) + +print("\nFull synchronization complete. All public repositories are now secured on the physical node.")