e2net/scripts
Archived
2
0
Fork 0
Code Issues Pull requests Packages Projects Releases Wiki Activity

add staging scripts

Browse source
This commit is contained in:
Laura Hausmann 2023-05-16 17:38:31 +02:00
parent 8461b31642
commit 340a6f368c
6 changed files with 41 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

0
cleanup-media.zsh → prod/cleanup-media.zsh
View file

0
cleanup-stale-minio-files.zsh → prod/cleanup-stale-minio-files.zsh
View file

0
fix-broken-media-cache-links.zsh → prod/fix-broken-media-cache-links.zsh
View file

22
staging/cleanup-media.zsh Executable file
View file

@ -0,0 +1,22 @@
#!/bin/zsh
date=$(date -d "-30 days" "+%F %T.000")
# Get all files so we can delete them later
rows=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\", \"thumbnailUrl\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network' AND \"createdAt\" < '$date' AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"avatarId\" = \"drive_file\".\"id\") AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"bannerId\" = \"drive_file\".\"id\")" -t -A --csv)
files=
for row (${(f)rows}) do
parts=("${(@s:,:)row}")
files+="${parts[1]}\n"
if [[ -n ${parts[2]} ]]; then
files+="${parts[2]}\n"
fi
done
# Replace file URLs with the remote URI
cleanup_query="UPDATE \"public\".\"drive_file\" SET \"isLink\" = 'TRUE', \"thumbnailUrl\" = NULL, \"url\" = \"uri\", \"accessKey\" = replace(\"accessKey\", 'drive/', ''), \"thumbnailAccessKey\" = replace(\"thumbnailAccessKey\", 'drive/', '') WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network' AND \"createdAt\" < '$date' AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"avatarId\" = \"drive_file\".\"id\") AND NOT EXISTS (SELECT 1 FROM \"user\" WHERE \"user\".\"bannerId\" = \"drive_file\".\"id\")"
echo "$cleanup_query" | sudo -iu postgres psql -d e2net-staging
# Cleanup object storage
echo "$files" | awk NF | sed 's/https:\/\/e2net-staging.cdn.ztn.sh/garage\/e2net-staging/g' | grep -P '\/e2net-staging\/drive\/.{16,}' | mcli rm --stdin --force

8
staging/cleanup-stale-minio-files.zsh Executable file
View file

@ -0,0 +1,8 @@
#!/bin/zsh
# Get all files so we can delete them later
files=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' UNION ALL SELECT \"thumbnailUrl\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE'" -t -A --csv | sed 's/https:\/\/e2net-staging.cdn.ztn.sh\/drive\///g' | awk NF | sort)
miniofiles=$(mcli ls garage/e2net-staging/drive | grep -Po '[^\s]*$' | grep -Pv '^webpublic-' | sort)
comm -23 <(echo "$miniofiles") <(echo "$files") | sed 's/^/garage\/e2net-staging\/drive\//g' | grep -P '\/e2net-staging\/drive\/.{16,}' | mcli rm --stdin --force

11
staging/fix-broken-media-cache-links.zsh Executable file
View file

@ -0,0 +1,11 @@
#!/bin/zsh
# Get all files so we can delete them later
files=$(sudo -iu postgres psql -d e2net-staging -c "SELECT \"url\" FROM \"public\".\"drive_file\" WHERE \"isLink\" = 'FALSE' AND \"storedInternal\" = 'FALSE' AND \"userHost\" IS NOT NULL AND \"userHost\" <> 'estrogen.network'" -t -A --csv | sed 's/https:\/\/e2net-staging.cdn.ztn.sh\/drive\///g' | awk NF | sort)
miniofiles=$(mcli ls garage/e2net-staging/drive | grep -Po '[^\s]*$' | grep -Pv '^webpublic-' | sort)
deadlinks=$(comm -13 <(echo "$miniofiles") <(echo "$files") | grep -P '.{16,}' | sed 's/^/https:\/\/e2net-staging.cdn.ztn.sh\/drive\//g')
while IFS= read -r file ; do
cleanup_query="UPDATE \"public\".\"drive_file\" SET \"isLink\" = 'TRUE', \"thumbnailUrl\" = NULL, \"url\" = \"uri\", \"accessKey\" = replace(\"accessKey\", 'drive/', ''), \"thumbnailAccessKey\" = replace(\"thumbnailAccessKey\", 'drive/', '') WHERE \"url\" = '$file'"
echo "$cleanup_query" | sudo -iu postgres psql -d e2net-staging
done <<< "$deadlinks"