Read the curated photo feeds from wss://nostr.kosmos.org, verify events, cache originals and thumbnails, and let users pick a wallpaper from the Photos section. Nature is the default feed; City Nights is opt-in. Adds the manual publishing scripts, real event fixtures, protocol documentation, and tests with stubbed network connections.
351 lines
12 KiB
Bash
Executable File
351 lines
12 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Verify originals, thumbnails, and creator attribution before anything is published.
|
|
set -euo pipefail
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
source "$SCRIPT_DIR/lib.sh"
|
|
|
|
require_publish_tools
|
|
require_cmd magick
|
|
validate_manifest
|
|
ensure_workdirs
|
|
|
|
ATTRIBUTION_ONLY=0
|
|
for arg in "$@"; do
|
|
case "$arg" in
|
|
--attribution-only) ATTRIBUTION_ONLY=1 ;;
|
|
--force|--yes|--dry-run) parse_common_args "$arg" ;;
|
|
*) die "unknown option: $arg" ;;
|
|
esac
|
|
done
|
|
|
|
THUMB_STATE="$STATE_DIR/thumbs.json"
|
|
[[ -f "$THUMB_STATE" ]] || die "no thumbnail state; run ./scripts/nostr/run.sh thumbnails first"
|
|
|
|
TMP_DIR="$WORK_DIR/preflight"
|
|
rm -rf "$TMP_DIR"
|
|
mkdir -p "$TMP_DIR"
|
|
|
|
fail_count=0
|
|
declare -A CONFIRMED_DOMAINS=()
|
|
pass() { info " ok: $1"; }
|
|
fail() { warn " FAIL: $1"; fail_count=$((fail_count + 1)); }
|
|
|
|
check_https() {
|
|
local label="$1" url="$2"
|
|
if [[ -z "$url" ]]; then
|
|
fail "$label is empty"
|
|
elif ! is_https "$url"; then
|
|
fail "$label is not https: $url"
|
|
else
|
|
pass "$label is https"
|
|
fi
|
|
}
|
|
|
|
chrome_bin() {
|
|
local bin
|
|
for bin in google-chrome-stable google-chrome chromium chromium-browser; do
|
|
if command -v "$bin" >/dev/null 2>&1; then
|
|
printf '%s\n' "$bin"
|
|
return 0
|
|
fi
|
|
done
|
|
return 1
|
|
}
|
|
|
|
probe_with_chrome() {
|
|
local url="$1" dest="$2" bin
|
|
bin="$(chrome_bin 2>/dev/null || true)"
|
|
[[ -n "$bin" ]] || return 1
|
|
mkdir -p "$WORK_DIR/chrome-profile"
|
|
if timeout 90 "$bin" --headless=new --no-sandbox --disable-gpu --disable-dev-shm-usage \
|
|
--no-first-run --no-default-browser-check \
|
|
--user-data-dir="$WORK_DIR/chrome-profile" \
|
|
--virtual-time-budget=15000 --dump-dom "$url" > "$dest" 2>/dev/null; then
|
|
[[ -s "$dest" ]] && return 0
|
|
fi
|
|
timeout 90 "$bin" --headless --no-sandbox --disable-gpu --disable-dev-shm-usage \
|
|
--no-first-run --no-default-browser-check \
|
|
--user-data-dir="$WORK_DIR/chrome-profile" \
|
|
--virtual-time-budget=15000 --dump-dom "$url" > "$dest" 2>/dev/null || return 1
|
|
[[ -s "$dest" ]]
|
|
}
|
|
|
|
looks_like_bot_challenge() {
|
|
local status="$1" body="$2"
|
|
[[ "$status" == "307" || "$status" == "401" || "$status" == "403" ]] && return 0
|
|
[[ -f "$body" ]] && grep -Eqi 'anubis|within\.website|Authorization required|Just a moment|Something went wrong' "$body" && return 0
|
|
return 1
|
|
}
|
|
|
|
is_hard_bot_block() {
|
|
local status="$1" body="$2"
|
|
[[ "$status" == "401" ]] && return 0
|
|
[[ -f "$body" ]] && grep -qi 'Authorization required' "$body" && return 0
|
|
return 1
|
|
}
|
|
|
|
recorded_status() {
|
|
jq -r --arg f "$1" --arg k "$2" '.photos[] | select(.file == $f) | .verified[$k].status // empty' "$MANIFEST"
|
|
}
|
|
|
|
recorded_name() {
|
|
jq -r --arg f "$1" --arg k "$2" '.photos[] | select(.file == $f) | .verified[$k].name // false' "$MANIFEST"
|
|
}
|
|
|
|
recorded_at() {
|
|
jq -r --arg f "$1" '.photos[] | select(.file == $f) | .verified.at // empty' "$MANIFEST"
|
|
}
|
|
|
|
accept_recorded() {
|
|
local label="$1" url="$2" creator="$3" file="$4" key="$5" need_name="$6"
|
|
local rec_status rec_name
|
|
rec_status="$(recorded_status "$file" "$key")"
|
|
rec_name="$(recorded_name "$file" "$key")"
|
|
if [[ "$rec_status" != "200" ]]; then
|
|
fail "$label: live check blocked and no recorded HTTP 200 verification"
|
|
return
|
|
fi
|
|
if [[ "$need_name" == "1" && "$rec_name" != "true" ]]; then
|
|
fail "$label: live check blocked and no recorded name verification"
|
|
return
|
|
fi
|
|
local domain
|
|
domain="$(printf '%s' "$url" | sed -E 's#^https?://([^/]+).*#\1#')"
|
|
if [[ "${CONFIRMED_DOMAINS[$domain]:-0}" == "1" ]]; then
|
|
pass "$label: previously confirmed for $domain"
|
|
return
|
|
fi
|
|
warn "$label: live check blocked by provider bot protection."
|
|
if [[ "$need_name" == "1" ]]; then
|
|
info " Recorded verification: $(recorded_at "$file") — HTTP 200 and name '$creator' present"
|
|
else
|
|
info " Recorded verification: $(recorded_at "$file") — HTTP 200"
|
|
fi
|
|
if [[ "$ASSUME_YES" == "1" ]]; then
|
|
CONFIRMED_DOMAINS[$domain]=1
|
|
pass "$label: accepted recorded verification (--yes)"
|
|
return
|
|
fi
|
|
if confirm "Open $url and confirm it still works${need_name:+ and shows '$creator'}? (asked once per domain)"; then
|
|
CONFIRMED_DOMAINS[$domain]=1
|
|
pass "$label: confirmed by user"
|
|
else
|
|
fail "$label: not confirmed"
|
|
fi
|
|
}
|
|
|
|
check_page() {
|
|
local label="$1" url="$2" creator="$3" file="$4" key="$5"
|
|
if [[ -z "$url" || "$url" != https://* ]]; then
|
|
fail "$label is missing or not https"
|
|
return
|
|
fi
|
|
local body="$TMP_DIR/page-$key-$(basename "$file")"
|
|
local status
|
|
status="$(curl -A "$USER_AGENT" -L -sS -o "$body" -w '%{http_code}' --max-time 60 --retry 2 "$url" 2>/dev/null || true)"
|
|
if [[ "$status" == "200" ]]; then
|
|
pass "$label returned HTTP 200"
|
|
if grep -Fqi -- "$creator" "$body"; then
|
|
pass "$label contains creator name '$creator'"
|
|
else
|
|
fail "$label does not contain creator name '$creator'"
|
|
fi
|
|
return
|
|
fi
|
|
if [[ "$status" == "404" || "$status" == "410" ]]; then
|
|
fail "$label returned HTTP $status: $url"
|
|
return
|
|
fi
|
|
if looks_like_bot_challenge "$status" "$body"; then
|
|
if ! is_hard_bot_block "$status" "$body"; then
|
|
local chrome_body="$TMP_DIR/chrome-$key-$(basename "$file")"
|
|
if probe_with_chrome "$url" "$chrome_body" && grep -Fqi -- "$creator" "$chrome_body"; then
|
|
pass "$label verified via headless browser"
|
|
return
|
|
fi
|
|
fi
|
|
accept_recorded "$label" "$url" "$creator" "$file" "$key" 1
|
|
return
|
|
fi
|
|
fail "$label returned HTTP $status: $url"
|
|
}
|
|
|
|
check_license() {
|
|
local label="$1" url="$2" file="$3"
|
|
if [[ -z "$url" || "$url" != https://* ]]; then
|
|
fail "$label is missing or not https"
|
|
return
|
|
fi
|
|
local body="$TMP_DIR/license-$(basename "$file")"
|
|
local status
|
|
status="$(curl -A "$USER_AGENT" -L -sS -o "$body" -w '%{http_code}' --max-time 60 --retry 2 "$url" 2>/dev/null || true)"
|
|
if [[ "$status" == "200" ]]; then
|
|
pass "$label returned HTTP 200"
|
|
return
|
|
fi
|
|
if looks_like_bot_challenge "$status" "$body"; then
|
|
accept_recorded "$label" "$url" "" "$file" license 0
|
|
return
|
|
fi
|
|
fail "$label returned HTTP $status: $url"
|
|
}
|
|
|
|
check_portfolio() {
|
|
local label="$1" url="$2" file="$3"
|
|
[[ -n "$url" ]] || return 0
|
|
if [[ "$url" != https://* ]]; then
|
|
fail "$label is not https"
|
|
return
|
|
fi
|
|
local body="$TMP_DIR/portfolio-$(basename "$file")"
|
|
local status
|
|
status="$(curl -A "$USER_AGENT" -L -sS -o "$body" -w '%{http_code}' --max-time 60 --retry 2 "$url" 2>/dev/null || true)"
|
|
if [[ "$status" == "200" ]]; then
|
|
pass "$label returned HTTP 200"
|
|
return
|
|
fi
|
|
if looks_like_bot_challenge "$status" "$body"; then
|
|
accept_recorded "$label" "$url" "" "$file" portfolio 0
|
|
return
|
|
fi
|
|
fail "$label returned HTTP $status: $url"
|
|
}
|
|
|
|
check_remote_image() {
|
|
local label="$1" url="$2" expected_sha="$3" expected_size="$4" dest="$5"
|
|
if [[ -z "$url" || "$url" != https://* ]]; then
|
|
fail "$label is missing or not https"
|
|
return
|
|
fi
|
|
if ! url_contains_sha "$url" "$expected_sha"; then
|
|
fail "$label URL does not contain its sha256: $url"
|
|
fi
|
|
local meta status content_type
|
|
meta="$(curl -A "$USER_AGENT" -L -sS -o "$dest" -w '%{http_code} %{content_type}' --max-time 120 --retry 2 "$url" 2>/dev/null || true)"
|
|
read -r status content_type <<<"$meta"
|
|
[[ "$status" == "200" ]] || {
|
|
fail "$label returned HTTP $status: $url"
|
|
return
|
|
}
|
|
pass "$label returned HTTP 200"
|
|
[[ "$content_type" == image/* ]] || fail "$label returned non-image content type: $content_type"
|
|
[[ "$content_type" == image/* ]] && pass "$label has image content type: $content_type"
|
|
local actual_sha actual_size
|
|
actual_sha="$(sha256_file "$dest")"
|
|
actual_size="$(file_size "$dest")"
|
|
[[ "$actual_sha" == "$expected_sha" ]] && pass "$label sha256 matches" || fail "$label sha256 mismatch (expected $expected_sha, got $actual_sha)"
|
|
if [[ -n "$expected_size" && "$expected_size" != "0" ]]; then
|
|
[[ "$actual_size" == "$expected_size" ]] && pass "$label size matches" || fail "$label size mismatch (expected $expected_size, got $actual_size)"
|
|
fi
|
|
}
|
|
|
|
info "Preflight: $MANIFEST"
|
|
info "Attribution only: $ATTRIBUTION_ONLY"
|
|
info ""
|
|
|
|
while IFS= read -r file; do
|
|
info "== $file =="
|
|
src="$ROOT/$file"
|
|
sha="$(photo_field "$file" sha256)"
|
|
size="$(photo_field "$file" size)"
|
|
dim="$(photo_field "$file" dim)"
|
|
mime="$(photo_field "$file" mime)"
|
|
url="$(photo_field "$file" url)"
|
|
feed="$(photo_field "$file" feed)"
|
|
title="$(photo_field "$file" title)"
|
|
alt="$(photo_field "$file" alt)"
|
|
content="$(photo_field "$file" content)"
|
|
creator="$(photo_field "$file" creator)"
|
|
profile="$(photo_field "$file" profile)"
|
|
source="$(photo_field "$file" source)"
|
|
license="$(photo_field "$file" license)"
|
|
license_url="$(photo_field "$file" licenseUrl)"
|
|
portfolio="$(photo_field "$file" portfolio)"
|
|
origin_id="$(photo_field "$file" originId)"
|
|
|
|
for field in sha size dim mime url feed title alt content creator profile source license license_url origin_id; do
|
|
[[ -n "${!field}" ]] || fail "required field '$field' is empty"
|
|
done
|
|
[[ -n "$(feed_field "$feed" title)" ]] || fail "unknown feed: $feed"
|
|
|
|
if [[ -f "$src" ]]; then
|
|
pass "local file exists"
|
|
[[ "$(sha256_file "$src")" == "$sha" ]] && pass "local sha256 matches manifest" || fail "local sha256 mismatch"
|
|
[[ "$(file_size "$src")" == "$size" ]] && pass "local size matches manifest" || fail "local size mismatch"
|
|
[[ "$(image_dim "$src")" == "$dim" ]] && pass "local dimensions match manifest" || fail "local dimensions mismatch"
|
|
[[ "$(file_mime "$src")" == "$mime" ]] && pass "local MIME matches manifest" || fail "local MIME mismatch"
|
|
else
|
|
fail "local file missing: $src"
|
|
fi
|
|
|
|
check_https "original url" "$url"
|
|
check_https "profile" "$profile"
|
|
check_https "source" "$source"
|
|
check_https "license url" "$license_url"
|
|
[[ -z "$portfolio" ]] || check_https "portfolio" "$portfolio"
|
|
|
|
check_remote_image "original" "$url" "$sha" "$size" "$TMP_DIR/$(basename "$file")"
|
|
|
|
if [[ "$ATTRIBUTION_ONLY" != "1" ]]; then
|
|
thumb_url="$(jq -r --arg f "$file" '.[$f].url // empty' "$THUMB_STATE")"
|
|
thumb_sha="$(jq -r --arg f "$file" '.[$f].sha256 // empty' "$THUMB_STATE")"
|
|
thumb_size="$(jq -r --arg f "$file" '.[$f].size // empty' "$THUMB_STATE")"
|
|
thumb_file="$(jq -r --arg f "$file" '.[$f].thumb // empty' "$THUMB_STATE")"
|
|
if [[ -z "$thumb_url" || -z "$thumb_sha" ]]; then
|
|
fail "thumbnail has not been uploaded"
|
|
else
|
|
check_https "thumbnail url" "$thumb_url"
|
|
check_remote_image "thumbnail" "$thumb_url" "$thumb_sha" "$thumb_size" "$TMP_DIR/thumb-$(basename "$file")"
|
|
if [[ -f "$thumb_file" ]]; then
|
|
[[ "$(sha256_file "$thumb_file")" == "$thumb_sha" ]] && pass "local thumbnail sha256 matches" || fail "local thumbnail sha256 mismatch"
|
|
else
|
|
fail "local thumbnail missing: $thumb_file"
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
check_page "profile page" "$profile" "$creator" "$file" profile
|
|
check_page "source page" "$source" "$creator" "$file" source
|
|
check_license "license url" "$license_url" "$file"
|
|
check_portfolio "portfolio" "$portfolio" "$file"
|
|
|
|
info ""
|
|
done < <(photo_files)
|
|
|
|
# Feed-level checks: every order entry exists (manifest validation) and, unless attribution-only,
|
|
# every photo has a thumbnail URL. Feed publication additionally requires published photo IDs.
|
|
while IFS= read -r feed; do
|
|
info "== feed $feed =="
|
|
cover="$(feed_field "$feed" cover)"
|
|
if [[ -n "$cover" && "$ATTRIBUTION_ONLY" != "1" ]]; then
|
|
cover_thumb="$(jq -r --arg f "$cover" '.[$f].url // empty' "$THUMB_STATE")"
|
|
[[ -n "$cover_thumb" ]] && pass "cover thumbnail is uploaded" || fail "cover thumbnail is not uploaded ($cover)"
|
|
fi
|
|
while IFS= read -r order_file; do
|
|
[[ -n "$(photo_field "$order_file" sha256)" ]] || fail "feed order references unknown photo: $order_file"
|
|
done < <(jq -r --arg f "$feed" '.feeds[] | select(.id == $f) | .order[]' "$MANIFEST")
|
|
info ""
|
|
done < <(feed_ids)
|
|
|
|
now="$(date -u +%Y-%m-%dT%H:%M:%SZ)"
|
|
ok=false
|
|
[[ "$fail_count" -eq 0 ]] && ok=true
|
|
jq -n \
|
|
--argjson ok "$ok" \
|
|
--argjson attributionOnly "$([[ "$ATTRIBUTION_ONLY" == "1" ]] && echo true || echo false)" \
|
|
--arg checkedAt "$now" \
|
|
--argjson failures "$fail_count" \
|
|
'{ok: $ok, attributionOnly: $attributionOnly, checkedAt: $checkedAt, failures: $failures}' \
|
|
> "$STATE_DIR/preflight.json"
|
|
|
|
if [[ "$fail_count" -gt 0 ]]; then
|
|
die "preflight failed with $fail_count issue(s)"
|
|
fi
|
|
|
|
info "Preflight passed."
|
|
if [[ "$ATTRIBUTION_ONLY" == "1" ]]; then
|
|
info "This was an attribution-only run; run the full preflight before publishing."
|
|
else
|
|
info "Publishing is allowed."
|
|
fi |