From c8d5427e16352925906e32381e88299a13236316 Mon Sep 17 00:00:00 2001
From: ij <ij@noreply.codeberg.org>
Date: Thu, 17 Aug 2023 19:25:27 +0000
Subject: [PATCH] added cron option and deactivation

Instead of deleting right away after the limit has been reached, the users will now be deactivated. The new "--cron" option can now be invoked more often (like hourly, daily, whatever) to delete the deactivated users. You can define how many accounts get deleted in each cron run. Default is 2 users.
Reason for this is to prevent too much load on the server when hundreds or even thousands of users get deleted.

Also there is some statistics at the end when the script runs.
---
 cleanup-mastodon-users.sh | 260 ++++++++++++++++++++++++++------------
 1 file changed, 182 insertions(+), 78 deletions(-)

diff --git a/cleanup-mastodon-users.sh b/cleanup-mastodon-users.sh
index 233b567..18c1036 100644
--- a/cleanup-mastodon-users.sh
+++ b/cleanup-mastodon-users.sh
@@ -10,9 +10,11 @@ UPPERLIMIT="7 months"
 TOOTCTL="~/live/bin/tootctl"
 
 set -f
+# set the following variables accordingly to your site
+# the admin will get a notification mail in BCC 
 
-# get some settings from the Mastodon config and write it to another file, so that we can source it and make use of it
 grep -e ^"DB_" -e ^"LOCAL_DOMAIN" /home/mastodon/live/.env.production > /home/mastodon/bin/cleanup-mastodon-users.conf
+
 . /home/mastodon/bin/cleanup-mastodon-users.conf
 
 # the following lines should be moved to a config file, eg. /usr/local/etc/cleanup_friendica.conf
@@ -23,23 +25,34 @@ mastodonpath="/home/mastodon/live"
 #DB_USER=mastodon
 #DB_PASS=
 
-site="<SITETITEL>"
-siteurl="<SITEURL>"
-siteadmin="<ADMINMAIL"
-sitefrom="no-reply@<SITEDOMAIN>"
-protectedusers="<space seperated list of protected accounts, eg: user1 user2>"
-sqlprotectedusers="<same, but with delimiters for SQL, eg:'user1', 'user2'>"
+site="Nerdculture.de"
+siteurl="https://nerdculture.de/"
+siteadmin="ij@bluespice.org"
+sitefrom="no-reply@nerdculture.de"
+protectedusers="ij kirschwipfel xmppcompliance order kirschwipfeltest"
+sqlprotectedusers="'ij', 'kirschwipfel', 'xmppcompliance', 'order', 'kirschwipfeltest'"
+limit_delete=2
+
+num_notified=0
+num_disabled=0
+num_deleted=0
+
+STARTDATE=$(date +"%d.%m.%Y %H:%M:%S")
 
 case $1 in
 	"--dry-run")
 		mode="dryrun"
 		;;
+	"--cron")
+		mode="cron"
+		;;
 	"--dowhatimean")
 		mode="hotrun"
 		;;
 	*)
 		echo "Usage: "
 		echo " --dry-run \t: make a dry-run, no deletion will be done, no mails are sent."
+		echo " --cron \t: delete deactivated users in a regularly cron run step by step to avoid mass flooding."
 		echo " --dowhatimean \t: add this option if you really want to delete users."
 		exit 0
 		;;
@@ -49,7 +62,7 @@ esac
 
 # make a list to be used for grep -E 
 protected=$(echo $protectedusers | sed 's/\"//g' | sed 's/\ /\\\|/g')
-echo $protected 
+#echo $protected 
 
 cd ${mastodonpath} || exit 0
 
@@ -60,7 +73,7 @@ Dear ${dispname},
 
 you have registered your account on ${siteurl} at ${registered} and last time you logged in was at ${lastlogin}. 
 
-If you want to continue to keep your Mastodon account on ${SITE} then please log in at least every 6 months via web browser to keep your account alive. Otherwise we assume that you don't want to use it anymore and will cancel your account 7 months after your last login. 
+If you want to continue to keep your Mastodon account on Nerdculture then please log in at least every 6 months via web browser to keep your account alive. Otherwise we assume that you don't want to use it anymore and will deactivate your account 7 months after your last login (and delete it later). 
 
 You can access your profile at ${profileurl} or you can cancel your account on your own when logged in at ${siteurl}removeme - however we would like to see you become an active user again and contribute to the Fediverse, but of course it's up to you.  
 
@@ -70,6 +83,24 @@ your ${site} admins
 EOF
 	) | sed 's/_/\ /g' | /usr/bin/mail -s "The Fediverse misses you, ${username}!" -r "${sitefrom}" -- "${usermail}"
 	# add '-b "$siteadmin"' before the "--" above to receive BCC mails 
+	#((num_notified++))
+}
+
+# notify user that the account has been deleted because of inactivity
+notifyUserDisable () {
+	( cat <<EOF  
+Dear ${dispname}, 
+
+you have registered your account on ${siteurl} at ${registered} and last time you logged in was at ${lastlogin}. 
+
+Since you haven't reacted to the previous mails and didn't login again, your account including all your data has now been deactivated and will be deleted at a (random) later time. 
+
+Sincerely,
+your ${site} admins
+    		
+EOF
+	) | sed 's/_/\ /g' | /usr/bin/mail -s "Your account ${username} on ${site} has been deleted!" -r "${sitefrom}" -- "${usermail}"
+	# add '-b "$siteadmin"' before the "--" above to receive BCC mails 
 }
 
 # notify user that the account has been deleted because of inactivity
@@ -89,76 +120,149 @@ EOF
 	# add '-b "$siteadmin"' before the "--" above to receive BCC mails 
 }
 
-for line in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||','||email) from accounts a, users u where domain is null and a.id=u.account_id and last_sign_in_at is null and u.created_at < now()-'2 weeks'::interval and username not in (${sqlprotectedusers})"| tr -d " "); do
-	#echo ${line}
-	username=$(echo ${line} | cut -f1 -d"," )
-	mail=$(echo ${line} | cut -f2 -d"," )
-	# if username is a protected user do nothing, else delete user
-	if [ -n "${protectedusers}" ]; then
-		pcheck=0
-		for s in $(echo ${protectedusers}) ; do
-			if [ "${s}" = "${username}" ]; then
-				pcheck=1
+# delete users that never logged in and never posted content
+# filtering for "weeks" will result in accounts with 2 weeks old accounts, 
+# filter for just "week" will do the same after 1 week.
+# same should apply to "month" and "months", but untested.
+#for username in $( ${mastodonpath}/bin/tootctl user list active -c 10000 | grep 'never.*never' | grep weeks | awk '{print $2}') ; do 
+
+
+# select a.id, username, email, current_sign_in_at from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at <'2019-01-01'
+
+case ${mode} in
+	"cron")
+		# get the total number of deactivated accounts 
+		# the intention is that you can better judge how often you need to invoke the cron option
+		# or by increasing the limit_delete variable
+		# the backlog queue shouldn't pile up but also not run empty to fast to reduce the load
+		num_deactivated_total=$(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select count(username) from accounts a, users u where disabled is true and a.id=u.account_id and current_sign_in_at > now()-'${UPPERLIMIT}'::interval and username not in ($sqlprotectedusers) limit ${limit_delete}" | tr -d " " )
+		# when there is less then 1 user to delete (=0) then exit
+		if [ ${num_deactivated_total} -lt 1 ]; then
+			exit 0
+		fi
+		echo "==================================="
+		echo "Number deactivated accounts: ${num_deactivated_total}"
+		echo "Deleting this many accounts: ${limit_delete}"
+		echo "==================================="
+		for u in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(current_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where disabled is true and a.id=u.account_id and current_sign_in_at > now()-'${UPPERLIMIT}'::interval and username not in ($sqlprotectedusers) order by current_sign_in_at limit ${limit_delete}" | tr -d " " ); do 
+			#echo ${u}
+			username=$(echo "${u}" | awk -F ";" '{print $1}')
+			dispname=$(echo "${u}" | awk -F ";" '{print $2}')
+			profileurl="https://nerdculture.de/@${username}"
+			usermail=$(echo "${u}" | awk -F ";" '{print $3}')
+			registered=$(echo "${u}" | awk -F ";" '{print $4}')
+			lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
+			# delete account when last login is older than 7 months and send mail about deletion
+			# you should copy & paste the text from 6 months for the first runs of this script
+			# and later change the text to a notification that the account has been deleted. 
+			# if username is a protected user do nothing, else delete user
+			echo -n "${username} : "
+			${mastodonpath}/bin/tootctl accounts delete "${username}" 
+			notifyUserDeletion
+			num_deleted=$((num_deleted+1))
+			RND=$(date +%s)
+			sec=$(( $RND %60 ))
+			ms=$(( $RND %23 ))
+			sleep ${sec}.${ms}			
+		done
+		;;
+	*)
+		# find & notify users that didn't logged in >6 months and send mail to log in again#psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||email) from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at is null and u.created_at < now()-'2 weeks'::interval" | tr -d " "
+		#for username in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select a.id, username, email, current_sign_in_at from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at <'2019-01-01'" )
+		for line in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||','||email) from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at is null and u.created_at < now()-'2 weeks'::interval and username not in (${sqlprotectedusers})"| tr -d " "); do
+			#echo ${line}
+			username=$(echo ${line} | cut -f1 -d"," )
+			mail=$(echo ${line} | cut -f2 -d"," )
+			# if username is a protected user do nothing, else delete user
+			if [ -n "${protectedusers}" ]; then
+				pcheck=0
+				for s in $(echo ${protectedusers}) ; do
+					if [ "${s}" = "${username}" ]; then
+						pcheck=1
+					fi
+				done
+				if [ ${pcheck} -eq 0 ]; then
+					echo "Delete unconfirmed user ${username}"
+					if [ "${mode}" = "hotrun" ]; then
+						${mastodonpath}/bin/tootctl accounts delete "${username}"
+					elif [ "${mode}" = "dryrun" ]; then
+						echo "${username}: skipped because of dryrun."
+					fi
+				fi
 			fi
 		done
-		if [ ${pcheck} -eq 0 ]; then
-			echo "Delete unconfirmed user ${username}"
-			if [ "${mode}" = "hotrun" ]; then
-				${mastodonpath}/bin/tootctl accounts delete "${username}"
-			elif [ "${mode}" = "dryrun" ]; then
-				echo "${username}: skipped because of dryrun."
-			fi
-		fi
-	fi
-done
+		#for u in $( ${mastodonpath}/bin/console user list active -c 10000 | grep -v '.*---.*' | sed 's/|/;/g' | tr  -s "\ " | sed 's/^;\ //g' | sed 's/\ ;\ /;/g' | sed 's/\ /_/g' | tail -n +2 | grep -i -v -E ${protected} ); do 
+		for mode2 in $(echo "warn disable"); do
+			#echo "mode2: $mode2"
+			case ${mode2} in
+				"warn")
+					#echo "in warn"
+					SQLSTATE="current_sign_in_at between now()-'${UPPERLIMIT}'::interval and now()-'${LOWERLIMIT}'::interval" 
+					;;
+				"disable")
+					#echo "in disable"
+					SQLSTATE="current_sign_in_at < now()-'${UPPERLIMIT}'::interval"
+					;;
+				#"delete")
+				#	#echo "in delete"
+				#	SQLSTATE="current_sign_in_at < now()-'${UPPERLIMIT}'::interval"
+				#	;;
+			esac
+			#echo "SQL: $SQLSTATE"
+			for u in $( psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(current_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where domain is null and a.id=u.account_id and ${SQLSTATE} and username not in ($sqlprotectedusers) order by current_sign_in_at" | tr -d " " ); do 
+				#echo ${u}
+				username=$(echo "${u}" | awk -F ";" '{print $1}')
+				dispname=$(echo "${u}" | awk -F ";" '{print $2}')
+				profileurl="https://nerdculture.de/@${username}"
+				usermail=$(echo "${u}" | awk -F ";" '{print $3}')
+				registered=$(echo "${u}" | awk -F ";" '{print $4}')
+				lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
+				case ${mode2} in
+					"warn")
+						if [ "${mode}" = "hotrun" ]; then
+							#echo -n "hotrun  "
+								notifyUser
+						elif [ "${mode}" = "dryrun" ]; then
+							echo "Check ${username}: notify skipped because of dryrun."
+						fi
+						num_notified=$((num_notified+1))
+						;;
+					"disable")
+						# delete account when last login is older than 7 months and send mail about deletion
+						# you should copy & paste the text from 6 months for the first runs of this script
+						# and later change the text to a notification that the account has been deleted. 
+						# if username is a protected user do nothing, else delete user
+						echo -n "${username} : "
+						if [ "${mode}" = "hotrun" ]; then
+							#echo -n "hotrun  "
+							#${mastodonpath}/bin/tootctl accounts delete "${username}" 
+							${mastodonpath}/bin/tootctl accounts modify "${username}" --disable
+							#notifyUserDeletion
+							notifyUserDisable
+							#echo "deleted."
+						elif [ "${mode}" = "dryrun" ]; then
+							echo "will be disabled, but is skipped because of dryrun."
+						fi
+						num_disabled=$((num_disabled+1))
+						;;
+				esac
+				RND=$(date +%s)
+				sec=$(( $RND %5 ))
+				ms=$(( $RND %23 ))
+				sleep ${sec}.${ms}
+			done
+		done
+		;;
+esac
 
-# find & notify users that didn't logged in >6 months and send mail to log in again
-for mode2 in $(echo "warn delete"); do
-	#echo "mode2: $mode2"
-	case ${mode2} in
-		"warn")
-			#echo "in warn"
-			SQLSTATE="last_sign_in_at between now()-'${UPPERLIMIT}'::interval and now()-'${LOWERLIMIT}'::interval" 
-			;;
-		"delete")
-			#echo "in delete"
-			SQLSTATE="last_sign_in_at < now()-'${UPPERLIMIT}'::interval"
-			;;
-	esac
-	#echo "SQL: $SQLSTATE"
-	for u in $( psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(last_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where domain is null and a.id=u.account_id and ${SQLSTATE} and username not in ($sqlprotectedusers) order by last_sign_in_at" | tr -d " " ); do 
-		#echo ${u}
-		username=$(echo "${u}" | awk -F ";" '{print $1}')
-		dispname=$(echo "${u}" | awk -F ";" '{print $2}')
-		profileurl="https://nerdculture.de/@${username}"
-		usermail=$(echo "${u}" | awk -F ";" '{print $3}')
-		registered=$(echo "${u}" | awk -F ";" '{print $4}')
-		lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
-		case ${mode2} in
-			"warn")
-				if [ "${mode}" = "hotrun" ]; then
-					#echo -n "hotrun  "
-    					notifyUser
-				elif [ "${mode}" = "dryrun" ]; then
-					echo "Check ${username}: notify skipped because of dryrun."
-				fi
-				;;
-			"delete")
-	   			# delete account when last login is older than 7 months and send mail about deletion
-    			# you should copy & paste the text from 6 months for the first runs of this script
-    			# and later change the text to a notification that the account has been deleted. 
-				# if username is a protected user do nothing, else delete user
-				echo -n "${username} : "
-				if [ "${mode}" = "hotrun" ]; then
-					#echo -n "hotrun  "
-					${mastodonpath}/bin/tootctl accounts delete "${username}" 
-					notifyUserDeletion
-					#echo "deleted."
-				elif [ "${mode}" = "dryrun" ]; then
-					echo "skipped because of dryrun."
-				fi
-				;;
-		esac
-	done
-done
+ENDDATE=$(date +"%d.%m.%Y %H:%M:%S")
+
+
+echo "==================================="
+echo "Starting time : ${STARTDATE}"
+echo "Ending time   : ${ENDDATE}"
+echo "Notified Users: $num_notified"
+echo "Disabled Users: $num_disabled"
+echo "Deleted Users : $num_deleted"
+echo "==================================="