Added configuration file and other changes

Reviewed-on: https://codeberg.org/Windfluechter/cleanup-mastodon-users.sh/pulls/3

README.md

@@ -1,9 +1,9 @@
 # cleanup-mastodon-users.sh
-Script to remove inactive users on your Mastodon server. This is a small cleanup script for Friendica node admins to help with forgotten or dead accounts. 
+Script to remove inactive users on your Mastodon server. This is a small cleanup script for Mastodon node admins to help with forgotten or dead accounts. 
 
 The script does two things: 
 1. delete all users after 2 weeks that haven't logged in at all and didn't post anything.
-2. search for accounts that haven't logged in for 6 months, send them a reminder mail and delete accounts that haven't logged within 7 months.
+2. search for accounts that haven't logged in for 6 months, send them a reminder mail, deactivate accounts that haven't logged within 7 months and delete those deactivated users in a `--cron` run. 
 
 # Installation
 * save the script in home directory of Mastodon (eg. ~/bin, not under /root, maybe /usr/local/bin)
@@ -16,6 +16,7 @@ The script does two things:
 ```
 Usage:
  --dry-run      : make a dry-run, no deletion will be done, no mails are sent.
+ --cron         : delete deactivated users in a regularly cron run step by step to avoid mass flooding.
  --dowhatimean  : add this option if you really want to delete users.
  ```
 
@@ -23,3 +24,7 @@ Usage:
 * test the script by using `--dry-run` argument with the script
 * you can change the mail command from `-b ${siteadmin} -- ${usermail}` to `-- ${siteadmin}` and comment out all lines with `bin/console user delete` statement for testing runs
 
+# Donations
+If you want to support my work, you can donate via LiberaPay: 
+
+[![Donate via LiberaPay](https://liberapay.com/assets/widgets/donate.svg)](https://liberapay.com/Windfluechter/donate) or on [Patreon](https://patreon.com/nerdculture_de)

cleanup-mastodon-users.sh

@@ -1,164 +1,277 @@
 #!/bin/sh
 
-# LOWERLIMIT="'6 months'" or "'24 weeks'"
-# UPPERLIMIT="'7 months'" or "'28 weeks'"
-# this defines the range of inactivity where users get notified
-# before the account will be deleted when UPPERLIMIT is surpassed.
-LOWERLIMIT="6 months"
-UPPERLIMIT="7 months"
-
-TOOTCTL="~/live/bin/tootctl"
-
 set -f
 
-# get some settings from the Mastodon config and write it to another file, so that we can source it and make use of it
-grep -e ^"DB_" -e ^"LOCAL_DOMAIN" /home/mastodon/live/.env.production > /home/mastodon/bin/cleanup-mastodon-users.conf
-. /home/mastodon/bin/cleanup-mastodon-users.conf
+if [ -f ${HOME}/.cleanup-mastodon-users.conf ]; then
+	. ${HOME}/.cleanup-mastodon-users.conf
+else 
+	(
+	cat <<EOF
+LOWERLIMIT="6 months"     # all accounts below 6 months inactivity are safe, start notifiying them when >6 months
+UPPERLIMIT="7 months"     # inactive accounts older than 7 months will be deactivated
+DELDELIMIT="1 year"       # all accounts not used within a year will get deleted
+LIVE_PATH="${HOME}/live/" # Path to live data from mastodon
+TOOTCTL="${HOME}/live/bin/tootctl"
+SITE="Mastodon"
+SITEADMIN="root@localhost"
+PROTECTEDUSERS="user foo bar baz"
+SQLPROTECTEDUSERS="'user', 'foo', 'bar', 'baz'"
+LIMIT_DELETE="2"
 
-# the following lines should be moved to a config file, eg. /usr/local/etc/cleanup_friendica.conf
-mastodonpath="/home/mastodon/live"
-#DB_HOST=127.0.0.1
-#DB_PORT=6432
-#DB_NAME=mastodon
-#DB_USER=mastodon
-#DB_PASS=
+EOF
+	) > ${HOME}/.cleanup-mastodon-users.conf
+	sleep 1
+	. ${HOME}/.cleanup-mastodon-users.conf
+fi
 
-site="<SITETITEL>"
-siteurl="<SITEURL>"
-siteadmin="<ADMINMAIL"
-sitefrom="no-reply@<SITEDOMAIN>"
-protectedusers="<space seperated list of protected accounts, eg: user1 user2>"
-sqlprotectedusers="<same, but with delimiters for SQL, eg:'user1', 'user2'>"
+#LOWERLIMIT="${LOWERLIMIT:-'6 months'}"     # all accounts below 6 months inactivity are safe, start notifiying them when >6 months
+#UPPERLIMIT="${UPPERLIMIT:-'7 months'}"     # inactive accounts older than 7 months will be deactivated
+#DELDELIMIT="${DELDELIMIT:-'1 year'}"       # all accounts not used within a year will get deleted
+#LIVE_PATH="${LIVE_PATH:-${HOME}/live/}" # Path to live data from mastodon
+#TOOTCTL="${TOOTCTL:-${HOME}/bin/tootctl}"
+#SITE="${SITE:-$LOCAL_DOMAIN}"
+#SITEADMIN="${SITEADMIN:-root@localhost}"
+#LIMIT_DELETE="${LIMIT_DELETE:-2}"
+
+
+cd "${LIVE_PATH}" || exit
+
+# shellcheck source=/dev/null
+source "${LIVE_PATH}/.env.production"
+
+run_tootctl() {
+	"${TOOTCTL}" "$@"
+}
+
+
+num_notified=0
+num_disabled=0
+num_deleted=0
+
+STARTDATE=$(date +"%d.%m.%Y %H:%M:%S")
 
 case $1 in
 	"--dry-run")
 		mode="dryrun"
 		;;
+	"--cron")
+		mode="cron"
+		;;
 	"--dowhatimean")
 		mode="hotrun"
 		;;
 	*)
 		echo "Usage: "
 		echo " --dry-run \t: make a dry-run, no deletion will be done, no mails are sent."
+		echo " --cron \t: delete deactivated users in a regularly cron run step by step to avoid mass flooding."
 		echo " --dowhatimean \t: add this option if you really want to delete users."
 		exit 0
 		;;
 esac
 
-#. /usr/local/etc/cleanup_friendica.conf
-
-# make a list to be used for grep -E 
-protected=$(echo $protectedusers | sed 's/\"//g' | sed 's/\ /\\\|/g')
-echo $protected 
-
-cd ${mastodonpath} || exit 0
+# make a list to be used for grep -E
+PROTECTED=$(echo "$PROTECTEDUSERS" | sed 's/\"//g' | sed 's/\ /\\\|/g')
+#echo $PROTECTED
 
 # notify the user that s/he needs to re-login after 6 months to prevent account deletion
-notifyUser () {
-	( cat <<EOF  
-Dear ${dispname}, 
+notifyUser() {
+	(
+		cat <<EOF
+Dear ${dispname},
 
-you have registered your account on ${siteurl} at ${registered} and last time you logged in was at ${lastlogin}. 
+you have registered your account on ${LOCAL_DOMAIN} at ${registered} and last time you logged in was at ${lastlogin}.
 
-If you want to continue to keep your Mastodon account on ${SITE} then please log in at least every 6 months via web browser to keep your account alive. Otherwise we assume that you don't want to use it anymore and will cancel your account 7 months after your last login. 
+If you want to continue to keep your Mastodon account on Nerdculture then please log in at least every 6 months via web browser to keep your account alive. Otherwise we assume that you don't want to use it anymore and will deactivate your account 7 months after your last login (and delete it later).
 
-You can access your profile at ${profileurl} or you can cancel your account on your own when logged in at ${siteurl}removeme - however we would like to see you become an active user again and contribute to the Fediverse, but of course it's up to you.  
+You can access your profile at ${profileurl} or you can cancel your account on your own when logged in at ${LOCAL_DOMAIN}removeme - however we would like to see you become an active user again and contribute to the Fediverse, but of course it's up to you.
 
 Sincerely,
-your ${site} admins
-    		
+your ${SITE} admins
+
 EOF
-	) | sed 's/_/\ /g' | /usr/bin/mail -s "The Fediverse misses you, ${username}!" -r "${sitefrom}" -- "${usermail}"
-	# add '-b "$siteadmin"' before the "--" above to receive BCC mails 
+	) | sed 's/_/\ /g' | /usr/bin/mail -s "The Fediverse misses you, ${username}!" -r "${SMTP_FROM_ADDRESS}" -- "${usermail}"
+	# add '-b "$SITEADMIN"' before the "--" above to receive BCC mails
+	#((num_notified++))
 }
 
 # notify user that the account has been deleted because of inactivity
-notifyUserDeletion () {
-	( cat <<EOF  
-Dear ${dispname}, 
+notifyUserDisable() {
+	(
+		cat <<EOF
+Dear ${dispname},
 
-you have registered your accoutn on ${siteurl} at ${registered} and last time you logged in was at ${lastlogin}. 
+you have registered your account on ${LOCAL_DOMAIN} at ${registered} and last time you logged in was at ${lastlogin}.
 
-Since you haven't reacted to the previous mails and didn't login again, your account including all your data has now been deleted. 
+Since you haven't reacted to the previous mails and didn't login again, your account including all your data has now been deactivated and will be deleted at a (random) later time.
 
 Sincerely,
-your ${site} admins
-    		
+your ${SITE} admins
+
 EOF
-	) | sed 's/_/\ /g' | /usr/bin/mail -s "Your account ${username} on ${site} has been deleted!" -r "${sitefrom}" -- "${usermail}"
-	# add '-b "$siteadmin"' before the "--" above to receive BCC mails 
+	) | sed 's/_/\ /g' | /usr/bin/mail -s "Your account ${username} on ${SITE} has been deleted!" -r "${SMTP_FROM_ADDRESS}" -- "${usermail}"
+	# add '-b "$SITEADMIN"' before the "--" above to receive BCC mails
 }
 
-for line in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||','||email) from accounts a, users u where domain is null and a.id=u.account_id and last_sign_in_at is null and u.created_at < now()-'2 weeks'::interval and username not in (${sqlprotectedusers})"| tr -d " "); do
-	#echo ${line}
-	username=$(echo ${line} | cut -f1 -d"," )
-	mail=$(echo ${line} | cut -f2 -d"," )
-	# if username is a protected user do nothing, else delete user
-	if [ -n "${protectedusers}" ]; then
-		pcheck=0
-		for s in $(echo ${protectedusers}) ; do
-			if [ "${s}" = "${username}" ]; then
-				pcheck=1
+# notify user that the account has been deleted because of inactivity
+notifyUserDeletion() {
+	(
+		cat <<EOF
+Dear ${dispname},
+
+you have registered your account on ${LOCAL_DOMAIN} at ${registered} and last time you logged in was at ${lastlogin}.
+
+Since you haven't reacted to the previous mails and didn't login again, your account including all your data has now been deleted.
+
+Sincerely,
+your ${SITE} admins
+
+EOF
+	) | sed 's/_/\ /g' | /usr/bin/mail -s "Your account ${username} on ${SITE} has been deleted!" -r "${SMTP_FROM_ADDRESS}" -- "${usermail}"
+	# add '-b "$SITEADMIN"' before the "--" above to receive BCC mails
+}
+
+# delete users that never logged in and never posted content
+# filtering for "weeks" will result in accounts with 2 weeks old accounts,
+# filter for just "week" will do the same after 1 week.
+# same should apply to "month" and "months", but untested.
+#for username in $( "${TOOTCTL}" user list active -c 10000 | grep 'never.*never' | grep weeks | awk '{print $2}') ; do
+
+# select a.id, username, email, current_sign_in_at from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at <'2019-01-01'
+
+case ${mode} in
+	"cron")
+		# get the total number of deactivated accounts
+		# the intention is that you can better judge how often you need to invoke the cron option
+		# or by increasing the limit_delete variable
+		# the backlog queue shouldn't pile up but also not run empty to fast to reduce the load
+		num_deactivated_overgrace=$(psql -U "${DB_USER}" -w -h "${DB_HOST}" -p "${DB_PORT}" -t "${DB_NAME}" -c "select count(username) from accounts a, users u where disabled is true and a.id=u.account_id and current_sign_in_at < now()-'${DELDELIMIT}'::interval and username not in ($SQLPROTECTEDUSERS)" | tr -d " " )
+		num_deactivated_total=$(psql -U "${DB_USER}" -w -h "${DB_HOST}" -p "${DB_PORT}" -t "${DB_NAME}" -c "select count(username) from accounts a, users u where disabled is true and a.id=u.account_id and username not in ($SQLPROTECTEDUSERS)" | tr -d " " )
+		# when there is less then 1 user to delete (=0) then exit
+		if [ "${num_deactivated_overgrace}" -lt 1 ]; then
+			exit 0
+		fi
+		echo "==================================="
+		echo "Total deactivated accounts: ${num_deactivated_total}"
+		echo "Number deactivated accounts over grace: ${num_deactivated_overgrace}"
+		echo "Deleting this many accounts: ${limit_delete}"
+		echo "==================================="
+		for u in $(psql -U "${DB_USER}" -w -h "${DB_HOST}" -p "${DB_PORT}" -t "${DB_NAME}" -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(current_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where disabled is true and a.id=u.account_id and current_sign_in_at < now()-'${DELDELIMIT}'::interval and username not in ($SQLPROTECTEDUSERS) order by current_sign_in_at limit ${limit_delete}" | tr -d " " ); do
+			#echo ${u}
+			username=$(echo "${u}" | awk -F ";" '{print $1}')
+			dispname=$(echo "${u}" | awk -F ";" '{print $2}')
+			profileurl="https://nerdculture.de/@${username}"
+			usermail=$(echo "${u}" | awk -F ";" '{print $3}')
+			registered=$(echo "${u}" | awk -F ";" '{print $4}')
+			lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
+			# delete account when last login is older than 7 months and send mail about deletion
+			# you should copy & paste the text from 6 months for the first runs of this script
+			# and later change the text to a notification that the account has been deleted.
+			# if username is a PROTECTED user do nothing, else delete user
+			echo -n "${username} : "
+			run_tootctl accounts delete "${username}"
+			notifyUserDeletion
+			num_deleted=$((num_deleted+1))
+			RND=$(date +%s)
+			sec=$(( $RND %60 ))
+			ms=$(( $RND %23 ))
+			sleep ${sec}.${ms}
+		done
+		;;
+	*)
+		# find & notify users that didn't logged in >6 months and send mail to log in again#psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||email) from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at is null and u.created_at < now()-'2 weeks'::interval" | tr -d " "
+		#for username in $(psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select a.id, username, email, current_sign_in_at from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at <'2019-01-01'" )
+		for line in $(psql -U "${DB_USER}" -w -h "${DB_HOST}" -p "${DB_PORT}" -t "${DB_NAME}" -c "select concat(username||','||email) from accounts a, users u where domain is null and a.id=u.account_id and current_sign_in_at is null and u.created_at < now()-'2 weeks'::interval and username not in (${SQLPROTECTEDUSERS})"| tr -d " "); do
+			#echo ${line}
+			username=$(echo "${line}" | cut -f1 -d"," )
+			mail=$(echo "${line}" | cut -f2 -d"," )
+			# if username is a PROTECTED user do nothing, else delete user
+			if [ -n "${PROTECTEDUSERS}" ]; then
+				pcheck=0
+				for s in ${PROTECTEDUSERS} ; do
+					if [ "${s}" = "${username}" ]; then
+						pcheck=1
+					fi
+				done
+				if [ ${pcheck} -eq 0 ]; then
+					echo "Delete unconfirmed user ${username}"
+					if [ "${mode}" = "hotrun" ]; then
+						run_tootctl accounts delete "${username}"
+					elif [ "${mode}" = "dryrun" ]; then
+						echo "${username}: skipped because of dryrun."
+					fi
+				fi
 			fi
 		done
-		if [ ${pcheck} -eq 0 ]; then
-			echo "Delete unconfirmed user ${username}"
-			if [ "${mode}" = "hotrun" ]; then
-				${mastodonpath}/bin/tootctl accounts delete "${username}"
-			elif [ "${mode}" = "dryrun" ]; then
-				echo "${username}: skipped because of dryrun."
-			fi
-		fi
-	fi
-done
+		#for u in $( ${LIVE_PATH}/bin/console user list active -c 10000 | grep -v '.*---.*' | sed 's/|/;/g' | tr  -s "\ " | sed 's/^;\ //g' | sed 's/\ ;\ /;/g' | sed 's/\ /_/g' | tail -n +2 | grep -i -v -E ${PROTECTED} ); do
+		for mode2 in $(echo "warn disable"); do
+			#echo "mode2: $mode2"
+			case ${mode2} in
+				"warn")
+					#echo "in warn"
+					SQLSTATE="current_sign_in_at between now()-'${UPPERLIMIT}'::interval and now()-'${LOWERLIMIT}'::interval"
+					;;
+				"disable")
+					#echo "in disable"
+					SQLSTATE="current_sign_in_at < now()-'${UPPERLIMIT}'::interval"
+					;;
+				#"delete")
+				#	#echo "in delete"
+				#	SQLSTATE="current_sign_in_at < now()-'${UPPERLIMIT}'::interval"
+				#	;;
+			esac
+			#echo "SQL: $SQLSTATE"
+			for u in $( psql -U "${DB_USER}" -w -h "${DB_HOST}" -p "${DB_PORT}" -t "${DB_NAME}" -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(current_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where domain is null and a.id=u.account_id and ${SQLSTATE} and username not in ($SQLPROTECTEDUSERS) order by current_sign_in_at" | tr -d " " ); do
+				#echo ${u}
+				username=$(echo "${u}" | awk -F ";" '{print $1}')
+				dispname=$(echo "${u}" | awk -F ";" '{print $2}')
+				profileurl="https://nerdculture.de/@${username}"
+				usermail=$(echo "${u}" | awk -F ";" '{print $3}')
+				registered=$(echo "${u}" | awk -F ";" '{print $4}')
+				lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
+				case ${mode2} in
+					"warn")
+						if [ "${mode}" = "hotrun" ]; then
+							#echo -n "hotrun  "
+								notifyUser
+						elif [ "${mode}" = "dryrun" ]; then
+							echo "Check ${username}: notify skipped because of dryrun."
+						fi
+						num_notified=$((num_notified+1))
+						;;
+					"disable")
+						# delete account when last login is older than 7 months and send mail about deletion
+						# you should copy & paste the text from 6 months for the first runs of this script
+						# and later change the text to a notification that the account has been deleted.
+						# if username is a PROTECTED user do nothing, else delete user
+						echo -n "${username} : "
+						if [ "${mode}" = "hotrun" ]; then
+							#echo -n "hotrun  "
+							#run_tootctl accounts delete "${username}"
+							run_tootctl accounts modify "${username}" --disable
+							#notifyUserDeletion
+							notifyUserDisable
+							#echo "deleted."
+						elif [ "${mode}" = "dryrun" ]; then
+							echo "will be disabled, but is skipped because of dryrun."
+						fi
+						num_disabled=$((num_disabled+1))
+						;;
+				esac
+				RND=$(date +%s)
+				sec=$(( $RND %5 ))
+				ms=$(( $RND %23 ))
+				sleep ${sec}.${ms}
+			done
+		done
+		;;
+esac
 
-# find & notify users that didn't logged in >6 months and send mail to log in again
-for mode2 in $(echo "warn delete"); do
-	#echo "mode2: $mode2"
-	case ${mode2} in
-		"warn")
-			#echo "in warn"
-			SQLSTATE="last_sign_in_at between now()-'${UPPERLIMIT}'::interval and now()-'${LOWERLIMIT}'::interval" 
-			;;
-		"delete")
-			#echo "in delete"
-			SQLSTATE="last_sign_in_at < now()-'${UPPERLIMIT}'::interval"
-			;;
-	esac
-	#echo "SQL: $SQLSTATE"
-	for u in $( psql -U ${DB_USER} -w -h ${DB_HOST} -p ${DB_PORT} -t ${DB_NAME} -c "select concat(username||';'||display_name||';'||email||';'||to_char(a.created_at, 'YYYY-MM-DD')||';'||to_char(last_sign_in_at,'YYYY-MM-DD')) from accounts a, users u where domain is null and a.id=u.account_id and ${SQLSTATE} and username not in ($sqlprotectedusers) order by last_sign_in_at" | tr -d " " ); do 
-		#echo ${u}
-		username=$(echo "${u}" | awk -F ";" '{print $1}')
-		dispname=$(echo "${u}" | awk -F ";" '{print $2}')
-		profileurl="https://nerdculture.de/@${username}"
-		usermail=$(echo "${u}" | awk -F ";" '{print $3}')
-		registered=$(echo "${u}" | awk -F ";" '{print $4}')
-		lastlogin=$(echo "${u}" | awk -F ";" '{print $5}')
-		case ${mode2} in
-			"warn")
-				if [ "${mode}" = "hotrun" ]; then
-					#echo -n "hotrun  "
-    					notifyUser
-				elif [ "${mode}" = "dryrun" ]; then
-					echo "Check ${username}: notify skipped because of dryrun."
-				fi
-				;;
-			"delete")
-	   			# delete account when last login is older than 7 months and send mail about deletion
-    			# you should copy & paste the text from 6 months for the first runs of this script
-    			# and later change the text to a notification that the account has been deleted. 
-				# if username is a protected user do nothing, else delete user
-				echo -n "${username} : "
-				if [ "${mode}" = "hotrun" ]; then
-					#echo -n "hotrun  "
-					${mastodonpath}/bin/tootctl accounts delete "${username}" 
-					notifyUserDeletion
-					#echo "deleted."
-				elif [ "${mode}" = "dryrun" ]; then
-					echo "skipped because of dryrun."
-				fi
-				;;
-		esac
-	done
-done
+ENDDATE=$(date +"%d.%m.%Y %H:%M:%S")
 
+echo "==================================="
+echo "Starting time : ${STARTDATE}"
+echo "Ending time   : ${ENDDATE}"
+echo "Notified Users: $num_notified"
+echo "Disabled Users: $num_disabled"
+echo "Deleted Users : $num_deleted"
+echo "==================================="