From 340a6f368ccc24e325999413b66fcc45839d4afd Mon Sep 17 00:00:00 2001
From: Laura Hausmann <laura@hausmann.dev>
Date: Tue, 16 May 2023 17:38:31 +0200
Subject: [PATCH] add staging scripts

---
 cleanup-media.zsh => prod/cleanup-media.zsh   |  0
 .../cleanup-stale-minio-files.zsh             |  0
 .../fix-broken-media-cache-links.zsh          |  0
 staging/cleanup-media.zsh                     | 22 +++++++++++++++++++
 staging/cleanup-stale-minio-files.zsh         |  8 +++++++
 staging/fix-broken-media-cache-links.zsh      | 11 ++++++++++
 6 files changed, 41 insertions(+)
 rename cleanup-media.zsh => prod/cleanup-media.zsh (100%)
 rename cleanup-stale-minio-files.zsh => prod/cleanup-stale-minio-files.zsh (100%)
 rename fix-broken-media-cache-links.zsh => prod/fix-broken-media-cache-links.zsh (100%)
 create mode 100755 staging/cleanup-media.zsh
 create mode 100755 staging/cleanup-stale-minio-files.zsh
 create mode 100755 staging/fix-broken-media-cache-links.zsh

diff --git a/cleanup-media.zsh b/prod/cleanup-media.zsh
similarity index 100%
rename from cleanup-media.zsh
rename to prod/cleanup-media.zsh
diff --git a/cleanup-stale-minio-files.zsh b/prod/cleanup-stale-minio-files.zsh
similarity index 100%
rename from cleanup-stale-minio-files.zsh
rename to prod/cleanup-stale-minio-files.zsh
diff --git a/fix-broken-media-cache-links.zsh b/prod/fix-broken-media-cache-links.zsh
similarity index 100%
rename from fix-broken-media-cache-links.zsh
rename to prod/fix-broken-media-cache-links.zsh
diff --git a/staging/cleanup-media.zsh b/staging/cleanup-media.zsh
new file mode 100755
index 0000000..6c86873
--- /dev/null
+++ b/staging/cleanup-media.zsh
@@ -0,0 +1,22 @@
+#!/bin/zsh
+
+date=$(date -d "-30 days" "+%F %T.000")
+
+# Get all files so we can delete them later
+rows=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\", \"thumbnailUrl\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network' AND \"createdAt\" < '$date' AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"avatarId\" = \"drive_file\".\"id\") AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"bannerId\" = \"drive_file\".\"id\")" -t -A --csv)
+files=
+
+for row (${(f)rows}) do
+  parts=("${(@s:,:)row}")
+  files+="${parts[1]}\n"
+  if [[ -n ${parts[2]} ]]; then
+    files+="${parts[2]}\n"
+  fi
+done
+
+# Replace file URLs with the remote URI
+cleanup_query="UPDATE \"public\".\"drive_file\" SET \"isLink\" = 'TRUE', \"thumbnailUrl\" = NULL, \"url\" = \"uri\", \"accessKey\" = replace(\"accessKey\", 'drive/', ''), \"thumbnailAccessKey\" = replace(\"thumbnailAccessKey\", 'drive/', '') WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network' AND \"createdAt\" < '$date' AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"avatarId\" = \"drive_file\".\"id\") AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"bannerId\" = \"drive_file\".\"id\")"
+echo "$cleanup_query" | sudo -iu postgres psql -d e2net-staging
+
+# Cleanup object storage
+echo "$files" | awk NF | sed 's/https:\/\/e2net-staging.cdn.ztn.sh/garage\/e2net-staging/g' | grep -P '\/e2net-staging\/drive\/.{16,}' | mcli rm --stdin --force
diff --git a/staging/cleanup-stale-minio-files.zsh b/staging/cleanup-stale-minio-files.zsh
new file mode 100755
index 0000000..7d35a94
--- /dev/null
+++ b/staging/cleanup-stale-minio-files.zsh
@@ -0,0 +1,8 @@
+#!/bin/zsh
+
+# Get all files so we can delete them later
+files=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' UNION ALL SELECT \"thumbnailUrl\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE'" -t -A --csv | sed 's/https:\/\/e2net-staging.cdn.ztn.sh\/drive\///g' | awk NF | sort)
+
+miniofiles=$(mcli ls garage/e2net-staging/drive | grep -Po '[^\s]*$' | grep -Pv '^webpublic-' | sort)
+
+comm -23 <(echo "$miniofiles") <(echo "$files") | sed 's/^/garage\/e2net-staging\/drive\//g' | grep -P '\/e2net-staging\/drive\/.{16,}' | mcli rm --stdin --force
diff --git a/staging/fix-broken-media-cache-links.zsh b/staging/fix-broken-media-cache-links.zsh
new file mode 100755
index 0000000..649a4a0
--- /dev/null
+++ b/staging/fix-broken-media-cache-links.zsh
@@ -0,0 +1,11 @@
+#!/bin/zsh
+
+# Get all files so we can delete them later
+files=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network'" -t -A --csv | sed 's/https:\/\/e2net-staging.cdn.ztn.sh\/drive\///g' | awk NF | sort)
+miniofiles=$(mcli ls garage/e2net-staging/drive | grep -Po '[^\s]*$' | grep -Pv '^webpublic-' | sort)
+deadlinks=$(comm -13 <(echo "$miniofiles") <(echo "$files") | grep -P '.{16,}' | sed 's/^/https:\/\/e2net-staging.cdn.ztn.sh\/drive\//g')
+
+while IFS= read -r file ; do
+	cleanup_query="UPDATE \"public\".\"drive_file\" SET \"isLink\" = 'TRUE', \"thumbnailUrl\" = NULL, \"url\" = \"uri\", \"accessKey\" = replace(\"accessKey\", 'drive/', ''), \"thumbnailAccessKey\" = replace(\"thumbnailAccessKey\", 'drive/', '') WHERE \"url\" = '$file'"
+	echo "$cleanup_query" | sudo -iu postgres psql -d e2net-staging
+done <<< "$deadlinks"