From ad78169a4a8543b40e4a2bcadbb0495da440cd40 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 14 Apr 2017 12:16:44 -0700 Subject: [PATCH 01/42] Copy log to remote location when done Prepping for some basic syncing --- launch.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/launch.pl b/launch.pl index 661a693..8d44c1a 100755 --- a/launch.pl +++ b/launch.pl @@ -87,6 +87,7 @@ } } while($nxtping <= time()); +system("scp -C $logf $remote_log/$usr.$remote_id.log"); unlock(); From 7b74f383f9337782760a1037cc2a2d926e855dbc Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 21 Apr 2017 13:28:55 -0700 Subject: [PATCH 02/42] Add basic remote sync functionality --- launch.pl | 66 ++++++++++++++++++++++++++++++++++---------- settings.pl.template | 7 +++++ 2 files changed, 59 insertions(+), 14 deletions(-) diff --git a/launch.pl b/launch.pl index 8d44c1a..ba5a65c 100755 --- a/launch.pl +++ b/launch.pl @@ -18,26 +18,42 @@ } # figure out the next ping after the last one that's in the log file -if(-e $logf) { - $lll = `tail -1 $logf`; # last log line - $lll =~ /^\s*(\d+)/; # parse out the timestamp for the last line, which better - $lstping = $1; # be equal to nextping@prevping of itself. - $tmp = nextping(prevping($lstping)); # NB: must call prevping before nextping - if($lstping == $tmp) { - $nxtping = nextping($lstping); - } else { - print "TagTime log file ($logf) has bad last line:\n$lll"; - $nxtping = prevping($launchTime); - } -} else { - $nxtping = prevping($launchTime); -} +$nxtping = parseping($logf); if(!lockn()) { print "Can't get lock. Exiting.\n" unless $quiet; exit(1); } # Don't wait if we can't get the lock. +if($nxtping < $launchTime-$retrothresh) { + # If we have a gap, first try to fill in with stuff from the most recent remote log + $lastremlog = `ssh $remote_server ls -tr1 $remote_path | tail -n1`; + chomp $lastremlog; + system(`scp -C $remote_server:$remote_path$lastremlog .`); + if(-e $logf) { + print "Filling in pings from remote file $lastremlog...\n" unless $quiet; + system(`cp $logf $logf.backup`); + open REMLOG, "<", $lastremlog; + open OURLOG, ">>", $logf; + $numfill = 0; + while() { + /^\s*(\d+)/; # parse out the timestamp for the current line + $curping = $1; + if($curping >= $nxtping) { + print OURLOG $_; + $numfill++; + } + } + print "Filled in $numfill pings\n" unless $quiet; + } else { + print "No previous logfile found, using entirety of $lastremlog\n" unless $quiet; + system(`cp $lastremlog $logf`) + } + + # Re-sync nxtping with any new loglines + $nxtping = parseping($logf); +} + my $editorFlag = 0; # First, if we missed any pings by more than $retrothresh seconds for no @@ -87,6 +103,7 @@ } } while($nxtping <= time()); +print "Backing up log to remote server...\n" unless $quiet; system("scp -C $logf $remote_log/$usr.$remote_id.log"); unlock(); @@ -132,6 +149,27 @@ sub editor { } } +sub parseping { + local $nxtping, $lastping; + local ($logf) = @_; + # figure out the next ping after the last one that's in the log file + if(-e $logf) { + $lll = `tail -1 $logf`; # last log line + $lll =~ /^\s*(\d+)/; # parse out the timestamp for the last line, which better + $lstping = $1; # be equal to nextping@prevping of itself. + $tmp = nextping(prevping($lstping)); # NB: must call prevping before nextping + if($lstping == $tmp) { + $nxtping = nextping($lstping); + } else { + print "TagTime log file ($logf) has bad last line:\n$lll"; + $nxtping = prevping($launchTime); + } + } else { + $nxtping = prevping($launchTime); + } + + return $nxtping; +} # SCHDEL (SCHEDULED FOR DELETION): (discussion and code for artificial gaps) # It can happen that 2 pings can both occur since we last checked (a minute diff --git a/settings.pl.template b/settings.pl.template index 4c4a3df..f020116 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -7,6 +7,13 @@ $path = "__PATH__"; # CHANGEME to your path to tagtime if($path !~ /\/$/) { $path.="/"; } $logf = "$path$usr.log"; # log file for pings +$remote_server = "__REMSERV__"; # CHANGEME to your backup server +$remote_path = "__REMPATH__"; # CHANGEME to the path on your backup server + if($remote_path !~ /\/$/) { $path.="/"; } +$remote_id = "__REMID__"; # CHANGEME to the unique tag for this machine on the backup server + +$remote_log = "$remote_server:$remote_path"; + # If you're using windows, you'll need cygwin and to set this flag to 1: $cygwin = __CYGWIN__; # CHANGEME to 1 if you're using windows/cygwin. From 6c4c0e7dc6f1487fe06f74ba27d2a9bec9fe2ac3 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 21 Apr 2017 14:02:39 -0700 Subject: [PATCH 03/42] Don't do any remote stuff if remote_id is empty --- launch.pl | 8 +++++--- settings.pl.template | 3 ++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/launch.pl b/launch.pl index ba5a65c..86e41e5 100755 --- a/launch.pl +++ b/launch.pl @@ -25,7 +25,7 @@ exit(1); } # Don't wait if we can't get the lock. -if($nxtping < $launchTime-$retrothresh) { +if($remote_id ne "" && $nxtping < $launchTime-$retrothresh) { # If we have a gap, first try to fill in with stuff from the most recent remote log $lastremlog = `ssh $remote_server ls -tr1 $remote_path | tail -n1`; chomp $lastremlog; @@ -103,8 +103,10 @@ } } while($nxtping <= time()); -print "Backing up log to remote server...\n" unless $quiet; -system("scp -C $logf $remote_log/$usr.$remote_id.log"); +if($remote_id ne "") { + print "Backing up log to remote server...\n" unless $quiet; + system("scp -C $logf $remote_log/$usr.$remote_id.log"); +} unlock(); diff --git a/settings.pl.template b/settings.pl.template index f020116..a70d63b 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -10,7 +10,8 @@ $logf = "$path$usr.log"; # log file for pings $remote_server = "__REMSERV__"; # CHANGEME to your backup server $remote_path = "__REMPATH__"; # CHANGEME to the path on your backup server if($remote_path !~ /\/$/) { $path.="/"; } -$remote_id = "__REMID__"; # CHANGEME to the unique tag for this machine on the backup server +$remote_id = ""; # CHANGEME to a unique tag for this machine on the backup server + # Leaving this blank will disable backup functionality $remote_log = "$remote_server:$remote_path"; From 3495dcfc76010ab93a7aea7fa82f19a404b5ba24 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 21 Apr 2017 14:11:26 -0700 Subject: [PATCH 04/42] Hypothetical fix for multiple users Had to switch to `cd` cause of the &&, but this also now does the `tail` on the remote server, which saves (a very small amount of) bandwidth --- launch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index 86e41e5..c697226 100755 --- a/launch.pl +++ b/launch.pl @@ -27,7 +27,7 @@ if($remote_id ne "" && $nxtping < $launchTime-$retrothresh) { # If we have a gap, first try to fill in with stuff from the most recent remote log - $lastremlog = `ssh $remote_server ls -tr1 $remote_path | tail -n1`; + $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1`; chomp $lastremlog; system(`scp -C $remote_server:$remote_path$lastremlog .`); if(-e $logf) { From 68bfc2c7664e3756c8a379d93335eb581823a2c9 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 21 Apr 2017 15:22:49 -0700 Subject: [PATCH 05/42] Commit fix for trailing ' Bah, meant to amend this to the last, not sure how it slipped through --- launch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index c697226..354a342 100755 --- a/launch.pl +++ b/launch.pl @@ -27,7 +27,7 @@ if($remote_id ne "" && $nxtping < $launchTime-$retrothresh) { # If we have a gap, first try to fill in with stuff from the most recent remote log - $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1`; + $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; chomp $lastremlog; system(`scp -C $remote_server:$remote_path$lastremlog .`); if(-e $logf) { From cb73435de6e072cad0dfc52f27aba392c426be20 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 21 Apr 2017 15:25:12 -0700 Subject: [PATCH 06/42] Attempt to fill from remote for any gap `$retrothresh` isn't applicable here, fixes syncing for quick (<`$retrothresh`) jumps between computers. Should be harmless, if there's updated pings it'll pull them in, otherwise will fill in or prompt as normal --- launch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index 354a342..2db4d49 100755 --- a/launch.pl +++ b/launch.pl @@ -25,7 +25,7 @@ exit(1); } # Don't wait if we can't get the lock. -if($remote_id ne "" && $nxtping < $launchTime-$retrothresh) { +if($remote_id ne "" && $nxtping < $launchTime) { # If we have a gap, first try to fill in with stuff from the most recent remote log $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; chomp $lastremlog; From 63b3bd9421b3858045bbdeeb149c62f7bab058b1 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 18 Mar 2020 22:28:18 -0700 Subject: [PATCH 07/42] Attempt to fix "have to kill launch" issue Now, closing the window without saving should backfill from remote, instead of backfilling locally and then uploading Haven't tested this yet though --- launch.pl | 80 ++++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 55 insertions(+), 25 deletions(-) diff --git a/launch.pl b/launch.pl index 2db4d49..39772f1 100755 --- a/launch.pl +++ b/launch.pl @@ -27,28 +27,7 @@ if($remote_id ne "" && $nxtping < $launchTime) { # If we have a gap, first try to fill in with stuff from the most recent remote log - $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; - chomp $lastremlog; - system(`scp -C $remote_server:$remote_path$lastremlog .`); - if(-e $logf) { - print "Filling in pings from remote file $lastremlog...\n" unless $quiet; - system(`cp $logf $logf.backup`); - open REMLOG, "<", $lastremlog; - open OURLOG, ">>", $logf; - $numfill = 0; - while() { - /^\s*(\d+)/; # parse out the timestamp for the current line - $curping = $1; - if($curping >= $nxtping) { - print OURLOG $_; - $numfill++; - } - } - print "Filled in $numfill pings\n" unless $quiet; - } else { - print "No previous logfile found, using entirety of $lastremlog\n" unless $quiet; - system(`cp $lastremlog $logf`) - } + fill_remote($nxtping); # Re-sync nxtping with any new loglines $nxtping = parseping($logf); @@ -74,6 +53,19 @@ launch($nxtping); # this shouldn't complete till you answer. } my($ts,$ln) = lastln(); + + # First, check to see if we have remote pings to fill in, if this computer + # was just sitting with a ping window up while they were being answered elsewhere + if($ts != $nxtping) { + my ($rts,$rln) = remoteln(); + if ($rts > $ts) { + print "$rts > $ts, filling from remote" unless $quiet; + fill_remote(nextping($ts)); + # re-read + ($ts, $ln) = lastln(); + } + } + if($ts != $nxtping) { # in case, eg, we closed the window w/o answering. # suppose there's a ping window waiting (call it ping 1), and while it's # sitting there unanswered another ping (ping 2) pings. then you kill @@ -110,15 +102,53 @@ unlock(); -# Returns the last line in the log but as a 2-element array +# Parses a log line as a 2-element array # consisting of timestamp and rest of the line. +sub parseln { + my ($x) = @_; + $x =~ /^\s*(\d+)\s*(.*)$/; + return ($1,$2); +} +# Returns the last line in the log as a 2-elm array sub lastln { my $x; open(L, $logf) or die "ERROR-lastln: Can't open log: $!"; $x = $_ while(); close(L); - $x =~ /^\s*(\d+)\s*(.*)$/; - return ($1,$2); + return parseln($x); +} + +# Returns the last line in the remote log as a 2-elm array +sub remoteln { + # If we have a gap, first try to fill in with stuff from the most recent remote log + $remote_line = `ssh $remote_server 'cd $remote_path && tail -n1 \$(ls -tr1 $usr.*.log | tail -n1)'`; + return parseln($remote_line); +} + +sub fill_remote { + my ($fill_from) = @_; + $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; + chomp $lastremlog; + system(`scp -C $remote_server:$remote_path$remlog .`); + if(-e $logf) { + print "Filling in pings from remote file $lastremlog...\n" unless $quiet; + system(`cp $logf $logf.backup`); + open REMLOG, "<", $lastremlog; + open OURLOG, ">>", $logf; + $numfill = 0; + while() { + /^\s*(\d+)/; # parse out the timestamp for the current line + $curping = $1; + if($curping >= $fill_from) { + print OURLOG $_; + $numfill++; + } + } + print "Filled in $numfill pings\n" unless $quiet; + } else { + print "No previous logfile found, using entirety of $lastremlog\n" unless $quiet; + system(`cp $lastremlog $logf`) + } } # Launch the tagtime pinger for the given time (in unix time). From 9158f519e623b6bb1d850c9d510b1d267a85f19c Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 18 Mar 2020 22:29:52 -0700 Subject: [PATCH 08/42] Set DISPLAY in launch This was causing ERR pings when running from systemd --- launch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index 39772f1..6b6b733 100755 --- a/launch.pl +++ b/launch.pl @@ -156,7 +156,7 @@ sub launch { my($t) = @_; my($sec,$min,$hour) = localtime($t); $sec = dd($sec); $min = dd($min); $hour = dd($hour); - #$ENV{DISPLAY} = ":0.0"; # have to set this explicitly if invoked by cron. + $ENV{DISPLAY} ||= ":0.0"; # have to set this explicitly if invoked by cron. if(!$quiet) { if(!defined($playsound)) { print STDERR "\a"; } else { system("$playsound") == 0 or print "SYSERR: $playsound\n"; } From 4e657ddf446617f71cae3da208d73757995bb0b8 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 10 Apr 2020 00:32:49 -0700 Subject: [PATCH 09/42] Fix variable name Missed this when moving into a function --- launch.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index 6b6b733..1659338 100755 --- a/launch.pl +++ b/launch.pl @@ -127,9 +127,10 @@ sub remoteln { sub fill_remote { my ($fill_from) = @_; + print "Backfilling with remote from $fill_from" unless $quiet; $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; chomp $lastremlog; - system(`scp -C $remote_server:$remote_path$remlog .`); + system(`scp -C $remote_server:$remote_path$lastremlog .`); if(-e $logf) { print "Filling in pings from remote file $lastremlog...\n" unless $quiet; system(`cp $logf $logf.backup`); From eaad7c239f001d2864197855dff3bd591224758f Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 12 Apr 2020 19:09:06 -0700 Subject: [PATCH 10/42] Add systemd service file --- tagtimed.service | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 tagtimed.service diff --git a/tagtimed.service b/tagtimed.service new file mode 100644 index 0000000..6325db8 --- /dev/null +++ b/tagtimed.service @@ -0,0 +1,12 @@ +[Unit] +Description=TagTime +Documentation= + +[Service] +Type=simple +ExecStart=/home/joel/bin/TagTime/tagtimed.pl +Restart=always +RestartSec=10 + +[Install] +WantedBy=default.target From 3199e5b33c9abe0491855c20f69413cc9b812b58 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Tue, 21 Apr 2020 12:57:56 -0700 Subject: [PATCH 11/42] Copy all log files, use ping time for most recent Also fix backtick-and-system nonsense --- launch.pl | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/launch.pl b/launch.pl index 1659338..b6d2d90 100755 --- a/launch.pl +++ b/launch.pl @@ -128,12 +128,15 @@ sub remoteln { sub fill_remote { my ($fill_from) = @_; print "Backfilling with remote from $fill_from" unless $quiet; - $lastremlog = `ssh $remote_server 'cd $remote_path && ls -tr1 $usr.*.log | tail -n1'`; + system("scp $remote_server:$remote_path$usr.*.log ."); + $lastremlog = `awk \ + '{if(\$1 > max) { max = \$1; latest = FILENAME}} END { print latest }'\ + $usr.*.log`; chomp $lastremlog; - system(`scp -C $remote_server:$remote_path$lastremlog .`); + if(-e $logf) { print "Filling in pings from remote file $lastremlog...\n" unless $quiet; - system(`cp $logf $logf.backup`); + system("cp $logf $logf.backup"); open REMLOG, "<", $lastremlog; open OURLOG, ">>", $logf; $numfill = 0; @@ -148,7 +151,7 @@ sub fill_remote { print "Filled in $numfill pings\n" unless $quiet; } else { print "No previous logfile found, using entirety of $lastremlog\n" unless $quiet; - system(`cp $lastremlog $logf`) + system("cp $lastremlog $logf") } } From 232de598ca55c3e7d739ae1dbe34fef7b8afff45 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 22 Apr 2020 15:27:51 -0700 Subject: [PATCH 12/42] Better logging --- launch.pl | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index b6d2d90..5e60242 100755 --- a/launch.pl +++ b/launch.pl @@ -62,7 +62,10 @@ print "$rts > $ts, filling from remote" unless $quiet; fill_remote(nextping($ts)); # re-read - ($ts, $ln) = lastln(); + ($ts,$ln) = lastln(); + print "New last timestamp: $ts" unless $quiet; + } else { + print "$rts <= $ts, nothing to fill from remote" unless $quiet; } } From 91aee713c3b060642d3e6c3aa4793bf7ee8b3c68 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 22 Apr 2020 15:28:05 -0700 Subject: [PATCH 13/42] Use tagtime-specific key/user Also move awk script into its own file and fix silly path error --- get_latest.awk | 1 + launch.pl | 10 ++++------ settings.pl.template | 7 ++++++- 3 files changed, 11 insertions(+), 7 deletions(-) create mode 100644 get_latest.awk diff --git a/get_latest.awk b/get_latest.awk new file mode 100644 index 0000000..d43487c --- /dev/null +++ b/get_latest.awk @@ -0,0 +1 @@ +{if($1 > max) { max = $1; latest = FILENAME}} END { print latest } diff --git a/launch.pl b/launch.pl index 5e60242..79e3601 100755 --- a/launch.pl +++ b/launch.pl @@ -100,7 +100,7 @@ if($remote_id ne "") { print "Backing up log to remote server...\n" unless $quiet; - system("scp -C $logf $remote_log/$usr.$remote_id.log"); + system("scp $remote_sshid -C $logf $remote_log$usr.$remote_id.log"); } unlock(); @@ -124,17 +124,15 @@ sub lastln { # Returns the last line in the remote log as a 2-elm array sub remoteln { # If we have a gap, first try to fill in with stuff from the most recent remote log - $remote_line = `ssh $remote_server 'cd $remote_path && tail -n1 \$(ls -tr1 $usr.*.log | tail -n1)'`; + $remote_line = `ssh $remote_sshid $remote_server 'cd $remote_path && tail -n1 \$(ls -tr1 $usr.*.log | tail -n1)'`; return parseln($remote_line); } sub fill_remote { my ($fill_from) = @_; print "Backfilling with remote from $fill_from" unless $quiet; - system("scp $remote_server:$remote_path$usr.*.log ."); - $lastremlog = `awk \ - '{if(\$1 > max) { max = \$1; latest = FILENAME}} END { print latest }'\ - $usr.*.log`; + system("scp $remote_sshid $remote_log$usr.*.log ."); + $lastremlog = `awk -f get_latest.awk $usr.*.log`; chomp $lastremlog; if(-e $logf) { diff --git a/settings.pl.template b/settings.pl.template index a70d63b..a1fd744 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -7,13 +7,18 @@ $path = "__PATH__"; # CHANGEME to your path to tagtime if($path !~ /\/$/) { $path.="/"; } $logf = "$path$usr.log"; # log file for pings +$remote_key = "id_tagtime"; # CHANGEME to the file contaning your tagtime key (if different) +$remote_user = "tagtime"; # CHANGEME to your backup user (if different) $remote_server = "__REMSERV__"; # CHANGEME to your backup server $remote_path = "__REMPATH__"; # CHANGEME to the path on your backup server - if($remote_path !~ /\/$/) { $path.="/"; } + if($remote_path !~ /\/$/) { $remote_path.="/"; } $remote_id = ""; # CHANGEME to a unique tag for this machine on the backup server # Leaving this blank will disable backup functionality +$remote_server = "$remote_user\@$remote_host"; $remote_log = "$remote_server:$remote_path"; +$remote_sshid = ""; + if ($remote_key ne "") { $remote_sshid = "-i $remote_key"; } # If you're using windows, you'll need cygwin and to set this flag to 1: $cygwin = __CYGWIN__; # CHANGEME to 1 if you're using windows/cygwin. From e0d030274661ee99ca2812206e2b26b005f03d51 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 22 Apr 2020 16:12:12 -0700 Subject: [PATCH 14/42] Add newlines and working directory --- launch.pl | 8 ++++---- tagtimed.service | 1 + 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/launch.pl b/launch.pl index 79e3601..05da0ef 100755 --- a/launch.pl +++ b/launch.pl @@ -59,13 +59,13 @@ if($ts != $nxtping) { my ($rts,$rln) = remoteln(); if ($rts > $ts) { - print "$rts > $ts, filling from remote" unless $quiet; + print "$rts > $ts, filling from remote\n" unless $quiet; fill_remote(nextping($ts)); # re-read ($ts,$ln) = lastln(); - print "New last timestamp: $ts" unless $quiet; + print "New last timestamp: $ts\n" unless $quiet; } else { - print "$rts <= $ts, nothing to fill from remote" unless $quiet; + print "$rts <= $ts, nothing to fill from remote\n" unless $quiet; } } @@ -130,7 +130,7 @@ sub remoteln { sub fill_remote { my ($fill_from) = @_; - print "Backfilling with remote from $fill_from" unless $quiet; + print "Backfilling with remote from $fill_from\n" unless $quiet; system("scp $remote_sshid $remote_log$usr.*.log ."); $lastremlog = `awk -f get_latest.awk $usr.*.log`; chomp $lastremlog; diff --git a/tagtimed.service b/tagtimed.service index 6325db8..9220a73 100644 --- a/tagtimed.service +++ b/tagtimed.service @@ -5,6 +5,7 @@ Documentation= [Service] Type=simple ExecStart=/home/joel/bin/TagTime/tagtimed.pl +WorkingDirectory=/home/joel/bin/TagTime/ Restart=always RestartSec=10 From 99ec2ffd864dd0deb4df1ac0dc02a4ed6d72d7ba Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Thu, 23 Apr 2020 20:24:54 -0700 Subject: [PATCH 15/42] Don't poison the generator --- launch.pl | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/launch.pl b/launch.pl index 05da0ef..d3e2595 100755 --- a/launch.pl +++ b/launch.pl @@ -60,10 +60,24 @@ my ($rts,$rln) = remoteln(); if ($rts > $ts) { print "$rts > $ts, filling from remote\n" unless $quiet; - fill_remote(nextping($ts)); + + $verify = nextping(prevping($ts)); # NB: must call prevping before nextping + if($ts == $verify) { + fill_remote(nextping($ts)); + } else { + print "Local file has a bad last line:\n$ln"; + $nxtping = prevping($launchTime); + } # re-read ($ts,$ln) = lastln(); - print "New last timestamp: $ts\n" unless $quiet; + + $verify = nextping(prevping($ts)); + if($ts == $verify) { + print "New last timestamp: $ts\n" unless $quiet; + } else { + print "Remote file has a bad last line:\n$ln"; + $nxtping = prevping($launchTime); + } } else { print "$rts <= $ts, nothing to fill from remote\n" unless $quiet; } From e6189632f23f32f73501a694bfeae44aa1dcba9a Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 29 Apr 2020 17:11:51 -0700 Subject: [PATCH 16/42] Fix remoteln() to also use ping timestamps Avoids depending on file modification time, which is unreliable --- launch.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index d3e2595..630dfcf 100755 --- a/launch.pl +++ b/launch.pl @@ -138,7 +138,7 @@ sub lastln { # Returns the last line in the remote log as a 2-elm array sub remoteln { # If we have a gap, first try to fill in with stuff from the most recent remote log - $remote_line = `ssh $remote_sshid $remote_server 'cd $remote_path && tail -n1 \$(ls -tr1 $usr.*.log | tail -n1)'`; + $remote_line = `ssh $remote_sshid $remote_server 'cd $remote_path && tail -n1 -q cincodenada.*.log | sort | tail -n1'`; return parseln($remote_line); } From 8ab6206dde472d9feec8691d8cc046987368071d Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 29 Apr 2020 17:29:48 -0700 Subject: [PATCH 17/42] Add a safety before deleting points from a beeminder log --- beeminder.pl | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/beeminder.pl b/beeminder.pl index ad0db3f..fbe56ea 100755 --- a/beeminder.pl +++ b/beeminder.pl @@ -200,9 +200,15 @@ $bh{$ts} = beemcreate($usr,$slug,$t, $p1*$ping, splur($p1,"ping").": ".$s1); #print "Created: $y $m $d ",$p1*$ping," \"$p1 pings: $s1\"\n"; } elsif($p0 > 0 && $p1 <= 0) { # on beeminder but not in tagtime log: DELETE - $ndel++; - $minus += $p0; - beemdelete($usr, $slug, $b); + print "Beeminder point not found in tagtime log! Delete? [y/N]"; + my $resp = ; + if(/^y/i) { + $ndel++; + $minus += $p0; + beemdelete($usr, $slug, $b); + } else { + print "Not deleting! Please fix your logs and run beeminder.pl manually!" + } #print "Deleted: $y $m $d ",$p0*$ping," \"$p0 pings: $s0 [bID:$b]\"\n"; } elsif($p0 != $p1 || $s0 ne $s1) { # bmndr & tagtime log differ: UPDATE $nchg++; From f7a0be56bd5156ba1593f92031f6847b3b6bd127 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 22 May 2020 17:03:20 -0700 Subject: [PATCH 18/42] Add more logging --- launch.pl | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/launch.pl b/launch.pl index 630dfcf..22bcabe 100755 --- a/launch.pl +++ b/launch.pl @@ -35,6 +35,7 @@ my $editorFlag = 0; +print "Filling in RETRO pings ($launchTime <=> $nxtping)\n" unless $quiet; # First, if we missed any pings by more than $retrothresh seconds for no # apparent reason, then assume the computer was off and auto-log them. while($nxtping < $launchTime-$retrothresh) { @@ -54,6 +55,8 @@ } my($ts,$ln) = lastln(); + print "Processing ping response ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + # First, check to see if we have remote pings to fill in, if this computer # was just sitting with a ping window up while they were being answered elsewhere if($ts != $nxtping) { @@ -83,6 +86,8 @@ } } + print "Checked from remote ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + if($ts != $nxtping) { # in case, eg, we closed the window w/o answering. # suppose there's a ping window waiting (call it ping 1), and while it's # sitting there unanswered another ping (ping 2) pings. then you kill @@ -100,6 +105,8 @@ $editorFlag = 1; } + print "Generated err pings ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + $lstping = $nxtping; $nxtping = nextping($nxtping); # Here's where we would add an artificial gap of $nxtping-$lstping. } From a9731200a9f1f40f9185e3c02aa333b154847f70 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Fri, 22 May 2020 17:03:34 -0700 Subject: [PATCH 19/42] Fix remote_host in template --- settings.pl.template | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.pl.template b/settings.pl.template index a1fd744..06a29ed 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -9,7 +9,7 @@ $logf = "$path$usr.log"; # log file for pings $remote_key = "id_tagtime"; # CHANGEME to the file contaning your tagtime key (if different) $remote_user = "tagtime"; # CHANGEME to your backup user (if different) -$remote_server = "__REMSERV__"; # CHANGEME to your backup server +$remote_host = "__REMHOST__"; # CHANGEME to your backup server $remote_path = "__REMPATH__"; # CHANGEME to the path on your backup server if($remote_path !~ /\/$/) { $remote_path.="/"; } $remote_id = ""; # CHANGEME to a unique tag for this machine on the backup server From b1765f73550d4c563ae2d2ba3d0bf327927d7aab Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 23 May 2020 15:25:08 -0700 Subject: [PATCH 20/42] Disable caching if using remote sync --- beeminder.pl | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/beeminder.pl b/beeminder.pl index fbe56ea..9fd74f8 100755 --- a/beeminder.pl +++ b/beeminder.pl @@ -46,10 +46,11 @@ # need to: 1. it doesn't exist or is empty; 2. any beeminder IDs are missing # from the cache file; 3. there are multiple datapoints for the same day. $bflag = (!-s $beef); -my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; # why bflag? +my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; my $bf5 = 0 # why bflag? $bf1 = 1 if $bflag; undef %remember; # remember which dates we've already seen in the cache file -if(open(B, "<$beef")) { +if($remote_id ne "") { $bflag = 1; $bf1 = 0; $bf5 = 1 } +elsif(open(B, "<$beef")) { while(my $l = ) { my($y,$m,$d,$v,$p,$c,$b) = ($l =~ / (\d+)\s+ # year @@ -102,6 +103,8 @@ print "Cache file has duplicate Bmndr IDs; recreating... "; } elsif($bf4) { print "Couldn't read cache file; recreating... "; + } elsif($bf5) { + print "Using remote sync, skipping cache file... "; } else { # this case is impossible print "Recreating Beeminder cache ($tmp)[$bf1$bf2$bf3$bf4]... "; } From 59ce5483d5b1ff9962b93589588c413f4441b2b5 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 24 May 2020 15:52:42 -0700 Subject: [PATCH 21/42] Fix semicolon --- beeminder.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/beeminder.pl b/beeminder.pl index 9fd74f8..6fd16f4 100755 --- a/beeminder.pl +++ b/beeminder.pl @@ -46,7 +46,7 @@ # need to: 1. it doesn't exist or is empty; 2. any beeminder IDs are missing # from the cache file; 3. there are multiple datapoints for the same day. $bflag = (!-s $beef); -my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; my $bf5 = 0 # why bflag? +my $bf1 = 0; my $bf2 = 0; my $bf3 = 0; my $bf4 = 0; my $bf5 = 0; # why bflag? $bf1 = 1 if $bflag; undef %remember; # remember which dates we've already seen in the cache file if($remote_id ne "") { $bflag = 1; $bf1 = 0; $bf5 = 1 } From 27fd7483817aee8c03b4575d1706d16535c35d27 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 20 Jun 2020 16:35:13 -0700 Subject: [PATCH 22/42] Parameterize scp/ssh commands for overriding --- launch.pl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/launch.pl b/launch.pl index 22bcabe..80a626a 100755 --- a/launch.pl +++ b/launch.pl @@ -121,7 +121,7 @@ if($remote_id ne "") { print "Backing up log to remote server...\n" unless $quiet; - system("scp $remote_sshid -C $logf $remote_log$usr.$remote_id.log"); + system("$scp_cmd -C $logf $remote_log$usr.$remote_id.log"); } unlock(); @@ -145,14 +145,14 @@ sub lastln { # Returns the last line in the remote log as a 2-elm array sub remoteln { # If we have a gap, first try to fill in with stuff from the most recent remote log - $remote_line = `ssh $remote_sshid $remote_server 'cd $remote_path && tail -n1 -q cincodenada.*.log | sort | tail -n1'`; + $remote_line = `$ssh_cmd $remote_server 'cd $remote_path && tail -n1 -q cincodenada.*.log | sort | tail -n1'`; return parseln($remote_line); } sub fill_remote { my ($fill_from) = @_; print "Backfilling with remote from $fill_from\n" unless $quiet; - system("scp $remote_sshid $remote_log$usr.*.log ."); + system("$scp_cmd $remote_log$usr.*.log ."); $lastremlog = `awk -f get_latest.awk $usr.*.log`; chomp $lastremlog; From 77d5801122d2350e8f4456dbd288113d1b7d7935 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 20 Jun 2020 16:45:30 -0700 Subject: [PATCH 23/42] Add initial basic testing harness --- test/.tagtimerc | 50 ++++++++++++++++++++++++++++++++++++++++++++ test/fakescp.pl | 11 ++++++++++ test/fakessh.pl | 3 +++ test/launch.pl | 55 +++++++++++++++++++++++++++++++++++++++++++++++++ test/logs/empty | 0 5 files changed, 119 insertions(+) create mode 100644 test/.tagtimerc create mode 100644 test/fakescp.pl create mode 100644 test/fakessh.pl create mode 100644 test/launch.pl create mode 100644 test/logs/empty diff --git a/test/.tagtimerc b/test/.tagtimerc new file mode 100644 index 0000000..4b4eab0 --- /dev/null +++ b/test/.tagtimerc @@ -0,0 +1,50 @@ +use File::Basename; + +$usr = "tagtime_test"; +$path = "$ENV{HOME}/../"; +$logf = "$path/test/logs/$usr.log"; # log file for pings + +$remote_key = "id_tagtime"; # CHANGEME to the file contaning your tagtime key (if different) +$remote_user = "tagtime"; # CHANGEME to your backup user (if different) +$remote_host = "fakehost"; # CHANGEME to your backup server +$remote_path = "fakepath"; # CHANGEME to the path on your backup server + if($remote_path !~ /\/$/) { $remote_path.="/"; } +$remote_id = "tester"; # CHANGEME to a unique tag for this machine on the backup server + # Leaving this blank will disable backup functionality + +$remote_server = "$remote_user\@$remote_host"; +$remote_log = "$remote_server:$remote_path"; +$remote_sshid = ""; + if ($remote_key ne "") { $remote_sshid = "-i $remote_key"; } + +$scp_cmd = "perl ${path}test/fakescp.pl"; +$ssh_cmd = "perl ${path}test/fakessh.pl"; + +# If you're using windows, you'll need cygwin and to set this flag to 1: +$cygwin = 0; # CHANGEME to 1 if you're using windows/cygwin. + +$ED = "__ED__ +"; # CHANGEME if you don't like vi (eg: /usr/bin/pico) +$XT = "__XT__"; # CHANGEME to your path to xterm + +# Get your personal Beeminder auth token (after signing in) from +# https://www.beeminder.com/api/v1/auth_token.json +$beemauth = "abc123"; # CHANGEME to your personal beeminder auth token + +%beeminder = (); + +$retrothresh = 60; + +$gap = 45*60; # Average number of seconds between pings (eg, 60*60 = 1 hour). + +$seed = 666; # For pings not in sync with others, change this (NB: > 0). + +$linelen = 79; # Try to keep log lines at most this long. + +$catchup = 0; # Whether it beeps for old pings, ie, should it beep a bunch + # of times in a row when the computer wakes from sleep. + +$enforcenums = 0; # Whether it forces you to include a number in your + # ping response (include tag non or nonXX where XX is day + # of month to override). This is for task editor integration. + +1; # When requiring a library in perl it has to return 1. diff --git a/test/fakescp.pl b/test/fakescp.pl new file mode 100644 index 0000000..fac2bab --- /dev/null +++ b/test/fakescp.pl @@ -0,0 +1,11 @@ +#!/usr/bin/env perl +my @argflags = []; +while(substr($ARGV[0], 0, 1) eq '-') { + $flag = shift @ARGV; + if(grep(/^$flag$/, @argflags)) { + $val = shift @ARGV; + } + print("Ignored flag $flag $val\n"); +} +my ($src, $dest) = @ARGV; +print "Would have copied from $src to $dest\n"; diff --git a/test/fakessh.pl b/test/fakessh.pl new file mode 100644 index 0000000..fa5693e --- /dev/null +++ b/test/fakessh.pl @@ -0,0 +1,3 @@ +#!/usr/bin/env perl +my ($server, $cmd) = @ARGV; +print "Would have run '$cmd' on $server\n"; diff --git a/test/launch.pl b/test/launch.pl new file mode 100644 index 0000000..d939c8b --- /dev/null +++ b/test/launch.pl @@ -0,0 +1,55 @@ +#!/usr/bin/env perl +use File::Basename; + +my $assertions_passed = 0; + +END { + print("$assertions_passed assertions passed!\n"); +} + +sub assert_numeq { + my ($expected, $actual) = @_; + assert($expected == $actual, "Expected $expected, got $actual"); +} + +sub assert { + my ($condition, $msg) = @_; + if($condition) { + $assertions_passed++; + return; + } + if (!$msg) { + my ($pkg, $file, $line) = caller(0); + open my $fh, "<", $file; + my @lines = <$fh>; + close $fh; + $msg = "$file:$line: " . $lines[$line - 1]; + } + die "Assertion failed: $msg"; +} + +$launchTime = 1592693841; # Birth of this test suite + +# Override home to be our test dir +$ENV{HOME} = "./" . dirname(__FILE__); +eval { + require "$ENV{HOME}/.tagtimerc"; +}; +if ($@) { + die "$0: $ENV{HOME}/.tagtimerc cannot be loaded ($!). Do you need to run install.py?\n" +} + +require "${path}util.pl"; + +my $lstping = prevping($launchTime); +my $nxtping = nextping($lstping); + +assert_numeq(1592691017, $lstping); +assert_numeq(1592696838, $nxtping); + +$newlog = "$path/test/logs/nonexistent.log"; +unlink $newlog; +$logf = "$path/test/logs/nonexistent.log"; + +$cmd = "${path}launch.pl"; # Catch up on any old pings. +system($cmd) == 0 or print "SYSERR: $cmd\n"; diff --git a/test/logs/empty b/test/logs/empty new file mode 100644 index 0000000..e69de29 From d3f3eb0c7d058c655e6062feaf009040de51bf33 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 20 Jun 2020 20:47:03 -0700 Subject: [PATCH 24/42] Start to deal with time faking, ugh --- launch.pl | 25 +++++++++++++++++++------ test/.tagtimerc | 8 ++++---- test/launch.pl | 25 +++++++++++++++++++++---- 3 files changed, 44 insertions(+), 14 deletions(-) diff --git a/launch.pl b/launch.pl index 80a626a..e0d2a69 100755 --- a/launch.pl +++ b/launch.pl @@ -3,7 +3,20 @@ # and/or launch ping.pl for the current ping. # This should be called by the daemon (tagtimed.pl) every time a ping is due. -$launchTime = time(); +my @faketimes; +if(exists($ENV{FAKETIMES})) { + @faketimes = split(",", $ENV{FAKETIMES}); +} + +sub mytime() { + if(exists($ENV{FAKETIMES})) { + print "Dispensing time $faketimes[0]\n"; + return shift @faketimes; + } + return time(); +} + +$launchTime = mytime(); require "$ENV{HOME}/.tagtimerc"; require "${path}util.pl"; @@ -46,8 +59,8 @@ # Next, ping for any pings in the last retrothresh seconds. do { - while($nxtping <= time()) { - if($nxtping < time()-$retrothresh) { + while($nxtping <= mytime()) { + if($nxtping < mytime()-$retrothresh) { slog(annotime("$nxtping afk RETRO", $nxtping)."\n"); $editorFlag = 1; } else { @@ -95,7 +108,7 @@ # ping but there will be nothing in the log yet for ping 2. perhaps # that's ok, just thinking out loud here... slog(annotime( - "$nxtping err [missed ping from ".ss(time()-$nxtping)." ago]", + "$nxtping err [missed ping from ".ss(mytime()-$nxtping)." ago]", $nxtping)."\n"); editor($logf,"TagTime Log Editor (unanswered pings logged as \"err\")"); $editorFlag = 0; @@ -117,7 +130,7 @@ # that's why we have the outer do-while loop here, to start over if # there are new pings in the past after we finish editing. } -} while($nxtping <= time()); +} while($nxtping <= mytime()); if($remote_id ne "") { print "Backing up log to remote server...\n" unless $quiet; @@ -243,7 +256,7 @@ sub parseping { # with zero gap. # if another ping is overdue, mind the gap! (ie delay the 2nd ping so as to # maintain the original gap betw them (but not more than retrothresh)): - #my $now = time(); + #my $now = mytime(); #my $eaten = $now - $prompt; # subtract amount of time eaten up # # answering last ping #if ($nxtping<$now && $nxtping>=$now-$retrothresh) { diff --git a/test/.tagtimerc b/test/.tagtimerc index 4b4eab0..d4a10e5 100644 --- a/test/.tagtimerc +++ b/test/.tagtimerc @@ -2,14 +2,14 @@ use File::Basename; $usr = "tagtime_test"; $path = "$ENV{HOME}/../"; -$logf = "$path/test/logs/$usr.log"; # log file for pings +$logf = "$path/test/logs/$ENV{LOGFILE}.log"; # log file for pings $remote_key = "id_tagtime"; # CHANGEME to the file contaning your tagtime key (if different) $remote_user = "tagtime"; # CHANGEME to your backup user (if different) $remote_host = "fakehost"; # CHANGEME to your backup server $remote_path = "fakepath"; # CHANGEME to the path on your backup server if($remote_path !~ /\/$/) { $remote_path.="/"; } -$remote_id = "tester"; # CHANGEME to a unique tag for this machine on the backup server +$remote_id = $ENV{REMOTEID}; # CHANGEME to a unique tag for this machine on the backup server # Leaving this blank will disable backup functionality $remote_server = "$remote_user\@$remote_host"; @@ -23,8 +23,8 @@ $ssh_cmd = "perl ${path}test/fakessh.pl"; # If you're using windows, you'll need cygwin and to set this flag to 1: $cygwin = 0; # CHANGEME to 1 if you're using windows/cygwin. -$ED = "__ED__ +"; # CHANGEME if you don't like vi (eg: /usr/bin/pico) -$XT = "__XT__"; # CHANGEME to your path to xterm +$ED = "perl ${path}test/fakeedit.pl"; # CHANGEME if you don't like vi (eg: /usr/bin/pico) +$XT = "perl ${path}test/faketerm.pl"; # CHANGEME to your path to xterm # Get your personal Beeminder auth token (after signing in) from # https://www.beeminder.com/api/v1/auth_token.json diff --git a/test/launch.pl b/test/launch.pl index d939c8b..d8c6623 100644 --- a/test/launch.pl +++ b/test/launch.pl @@ -28,8 +28,6 @@ sub assert { die "Assertion failed: $msg"; } -$launchTime = 1592693841; # Birth of this test suite - # Override home to be our test dir $ENV{HOME} = "./" . dirname(__FILE__); eval { @@ -41,15 +39,34 @@ sub assert { require "${path}util.pl"; -my $lstping = prevping($launchTime); +my $baseTime = 1592693841; # Birth of this test suite + +my $lstping = prevping($baseTime); my $nxtping = nextping($lstping); assert_numeq(1592691017, $lstping); assert_numeq(1592696838, $nxtping); +################## +# Test initializing a new log +################## + $newlog = "$path/test/logs/nonexistent.log"; unlink $newlog; -$logf = "$path/test/logs/nonexistent.log"; +$ENV{LOGFILE} = "nonexistent"; + +# This is finicky because we do a lot of time checks +# One check for $launchTime at the beginning, then loops +# Each inner loop we check: +# - beginning of loop +# - retrothresh +# - err (if we closed launch window) +# Plus one for outer loop (after filling in any RETRO) +$ENV{FAKETIMES} = join(",", ( + $lstping + 10, + $lstping + 11, # first $nxtping check + $lstping + 12, # retrothresh +)); $cmd = "${path}launch.pl"; # Catch up on any old pings. system($cmd) == 0 or print "SYSERR: $cmd\n"; From 72818243539cf2269a85134d5dab6d15efc76529 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 20 Jun 2020 20:52:12 -0700 Subject: [PATCH 25/42] Move time dispenser into util --- launch.pl | 18 +++--------------- ping.pl | 8 ++++---- test/launch.pl | 1 + util.pl | 12 ++++++++++++ 4 files changed, 20 insertions(+), 19 deletions(-) diff --git a/launch.pl b/launch.pl index e0d2a69..eb83422 100755 --- a/launch.pl +++ b/launch.pl @@ -3,24 +3,11 @@ # and/or launch ping.pl for the current ping. # This should be called by the daemon (tagtimed.pl) every time a ping is due. -my @faketimes; -if(exists($ENV{FAKETIMES})) { - @faketimes = split(",", $ENV{FAKETIMES}); -} - -sub mytime() { - if(exists($ENV{FAKETIMES})) { - print "Dispensing time $faketimes[0]\n"; - return shift @faketimes; - } - return time(); -} - -$launchTime = mytime(); - require "$ENV{HOME}/.tagtimerc"; require "${path}util.pl"; +$launchTime = mytime(); + my $args = join(' ', @ARGV); # supported arguments: test, quiet my $test = ($args =~ /\btest\b/); my $quiet = ($args =~ /\bquiet\b/); @@ -223,6 +210,7 @@ sub editor { sub parseping { local $nxtping, $lastping; local ($logf) = @_; + print("Launch time: $launchTime\n"); # figure out the next ping after the last one that's in the log file if(-e $logf) { $lll = `tail -1 $logf`; # last log line diff --git a/ping.pl b/ping.pl index 6ecebfc..7bcc6f4 100755 --- a/ping.pl +++ b/ping.pl @@ -12,12 +12,12 @@ Term::ANSIColor->import(':constants'); }; -my $pingTime = time(); -my $autotags = ""; - require "$ENV{HOME}/.tagtimerc"; require "${path}util.pl"; +my $pingTime = mytime(); +my $autotags = ""; + my $tskf = "$path$usr.tsk"; my $eflag = 0; # if any problems then prompt before exiting @@ -27,7 +27,7 @@ $t = shift; if(!defined($t)) { $autotags .= " UNSCHED"; - $t = time(); + $t = mytime(); } # Can't lock the same lockfile here since launch.pl will have the lock! diff --git a/test/launch.pl b/test/launch.pl index d8c6623..235c0ee 100644 --- a/test/launch.pl +++ b/test/launch.pl @@ -66,6 +66,7 @@ sub assert { $lstping + 10, $lstping + 11, # first $nxtping check $lstping + 12, # retrothresh + $lstping + 13, # pingtime )); $cmd = "${path}launch.pl"; # Catch up on any old pings. diff --git a/util.pl b/util.pl index d6d0604..f1b6c78 100644 --- a/util.pl +++ b/util.pl @@ -359,6 +359,18 @@ sub padr { return timelocal($second,$minute,$hour,$day,$month-1,$year); } +# Testing mocks +sub mytime() { + if(exists($ENV{FAKETIMES})) { + my @faketimes = split(",", $ENV{FAKETIMES}); + $next_time = shift @faketimes; + $ENV{FAKETIMES} = join(",", @faketimes); + print "Dispensing time $next_time\n"; + return shift @faketimes; + } + return time(); +} + 1; # perl wants this for libraries imported with 'require'. From 143d29b3c380571c9d02a71b4354068e4ac5386f Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Wed, 24 Jun 2020 12:45:22 -0700 Subject: [PATCH 26/42] Add new variables to template --- settings.pl.template | 3 +++ 1 file changed, 3 insertions(+) diff --git a/settings.pl.template b/settings.pl.template index 06a29ed..38b26e5 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -20,6 +20,9 @@ $remote_log = "$remote_server:$remote_path"; $remote_sshid = ""; if ($remote_key ne "") { $remote_sshid = "-i $remote_key"; } +$scp_cmd = "scp $remote_sshid"; +$ssh_cmd = "ssh $remote_sshid"; + # If you're using windows, you'll need cygwin and to set this flag to 1: $cygwin = __CYGWIN__; # CHANGEME to 1 if you're using windows/cygwin. From 8b2a4abf20847ff81dc3ada16e4feb3614830db0 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Mon, 20 Jul 2020 08:46:45 -0700 Subject: [PATCH 27/42] Don't overwrite our own log --- launch.pl | 3 +++ 1 file changed, 3 insertions(+) diff --git a/launch.pl b/launch.pl index eb83422..8b0c28d 100755 --- a/launch.pl +++ b/launch.pl @@ -153,6 +153,9 @@ sub fill_remote { my ($fill_from) = @_; print "Backfilling with remote from $fill_from\n" unless $quiet; system("$scp_cmd $remote_log$usr.*.log ."); + # Remove our log, we are source of truth for it + # Otherwise we overwrite our own edits, bleh + unlink "$usr.$remote_id.log"; $lastremlog = `awk -f get_latest.awk $usr.*.log`; chomp $lastremlog; From a80c26b5ea04b556975dc12180a908c58eba06fe Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Mon, 20 Jul 2020 08:47:02 -0700 Subject: [PATCH 28/42] Delete all Beeminder points with Y --- beeminder.pl | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/beeminder.pl b/beeminder.pl index 6fd16f4..a294c86 100755 --- a/beeminder.pl +++ b/beeminder.pl @@ -185,6 +185,7 @@ my $minus = 0; # total number of pings decreased from what's on beeminder my $plus = 0; # total number of pings increased from what's on beeminder my $ii = 0; +my $delall = 0; for(my $t = daysnap($start)-86400; $t <= daysnap($end)+86400; $t += 86400) { my($y,$m,$d) = dt($t); my $ts = "$y-$m-$d"; @@ -203,12 +204,16 @@ $bh{$ts} = beemcreate($usr,$slug,$t, $p1*$ping, splur($p1,"ping").": ".$s1); #print "Created: $y $m $d ",$p1*$ping," \"$p1 pings: $s1\"\n"; } elsif($p0 > 0 && $p1 <= 0) { # on beeminder but not in tagtime log: DELETE - print "Beeminder point not found in tagtime log! Delete? [y/N]"; - my $resp = ; - if(/^y/i) { + my $resp = 'y'; + unless($delall) { + print "Beeminder point not found in tagtime log! Delete? [y/N]"; + $resp = ; + } + if($resp =~ /^y/i or $delall) { $ndel++; $minus += $p0; beemdelete($usr, $slug, $b); + if($resp =~ /^Y/) { $delall = 1; } } else { print "Not deleting! Please fix your logs and run beeminder.pl manually!" } From 940b660a2ec4adc18f2ec1d0e4996f06ce8ce61f Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 16:51:26 -0700 Subject: [PATCH 29/42] Make merge a library --- merge.pl | 143 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 76 insertions(+), 67 deletions(-) diff --git a/merge.pl b/merge.pl index 07d57b5..12da119 100755 --- a/merge.pl +++ b/merge.pl @@ -2,20 +2,20 @@ # Merge the tagtime logs given on the command line; output to stdout. # If only one log file is given this just fills in any missing pings autotagged # with MISSING and autotags pings that shouldn't be there with UNSCHED. -# NB: The rest of this is not fully implemented yet! -# Currently just concatenates the tags from each log file. -# Eventual spec follows, and in the meantime you can use it with a single +# NB: The rest of this is not fully implemented yet! +# Currently just concatenates the tags from each log file. +# Eventual spec follows, and in the meantime you can use it with a single # log file to just sanity check it... # If multiple log files are given this will properly merge them, like if you # use tagtime on multiple computers. -# Any ping that, according to the ping schedule, is missing from all the given -# logs will be added with the autotag MISSING and any pings present in any of +# Any ping that, according to the ping schedule, is missing from all the given +# logs will be added with the autotag MISSING and any pings present in any of # the logs that shouldn't be there (again, according to the ping schedule) will # have the autotag UNSCHED appended. -# For each outputted ping with timestamp t, include the union of the tags with +# For each outputted ping with timestamp t, include the union of the tags with # timestamp t in all the given log files, ignoring log files that are tagged # only with autotags at time t. Unless *all* the log files are tagged only with -# autotags at time t, in wich case go ahead and do the union like normal. +# autotags at time t, in wich case go ahead and do the union like normal. # Autotags are {MISSING, UNSCHED, RETRO, afk, off, err}. # The earliest timestamp outputted is the earliest timestamp in all the logs. # The latest timestamp outputted is the max of now and the latest in the logs. @@ -26,7 +26,7 @@ # missflag = true # let @p = {}, a list of ping responses for each log file # for each log file l: -# if tags{l+t} not empty: missflag = false +# if tags{l+t} not empty: missflag = false # push(@p, tags{l+t}) # if sch{t} and missflag: push(@p, "MISSING") # if not sch{t}: push(@p, "UNSCHED") @@ -35,69 +35,78 @@ BEGIN { require "$ENV{HOME}/.tagtimerc"; } require "util.pl"; -die "USAGE: $0 logfile+\n" if @ARGV < 1; - -my $e = 0; # number of lines with parse errors -my $errstr = ""; # concatenation of bad lines from log files -my $earliest = -1; # earliest timestamp in all the log files -my $latest = 0; # latest timestamp in all the log files -my %th; # maps logfile+timestamp to tags for that log for that ping -my %alltimes; # maps all timestamps to 1 -for my $logfile (@ARGV) { - open(LOG, $logfile) or die; - $prevts = 0; # remember the previous timestamp - while($line = ) { - if(!parsable($line)) { - $e++; - $errstr .= $line; - next; - } - my @tags = split(/\s+/, $line); - my $ts = shift(@tags); - if($ts <= $prevts) { - $e++; - $errstr .= "NON-MONOTONE:\n$line"; - next; - } - $prevts = $ts; - if($ts < $earliest || $earliest == -1) { $earliest = $ts; } - if($ts > $latest) { $latest = $ts; } - $line =~ s/^\d+\s+//; - chomp($line); - $th{$logfile.$ts} = $line; - $alltimes{$ts} = 1; - } - close(LOG); -} +sub merge { + my $e = 0; # number of lines with parse errors + my $errstr = ""; # concatenation of bad lines from log files + my $earliest = -1; # earliest timestamp in all the log files + my $latest = 0; # latest timestamp in all the log files + my %th; # maps logfile+timestamp to tags for that log for that ping + my %alltimes; # maps all timestamps to 1 + for my $logfile (@_) { + open(LOG, $logfile) or die; + $prevts = 0; # remember the previous timestamp + while($line = ) { + if(!parsable($line)) { + $e++; + $errstr .= $line; + next; + } + my @tags = split(/\s+/, $line); + my $ts = shift(@tags); + if($ts <= $prevts) { + $e++; + $errstr .= "NON-MONOTONE:\n$line"; + next; + } + $prevts = $ts; + if($ts < $earliest || $earliest == -1) { $earliest = $ts; } + if($ts > $latest) { $latest = $ts; } + $line =~ s/^\d+\s+//; + chomp($line); + $th{$logfile.$ts} = $line; + $alltimes{$ts} = 1; + } + close(LOG); + } -if($e>0) { - print "Errors in log file(s): $e. ", - "They have to be fixed before this script can run:\n"; - print "\n$errstr"; - exit(1); -} + if($e>0) { + print "Errors in log file(s): $e. ", + "They have to be fixed before this script can run:\n"; + print "\n$errstr"; + exit(1); + } -my $now = time(); -if($now > $latest) { $latest = $now; } -my %sch; # maps timestamps to whether they are a scheduled pings -my $i = prevping($earliest); -$i = nextping($i); -while($i <= $latest) { - $sch{$i} = 1; - $alltimes{$i} = 1; + my $now = time(); + if($now > $latest) { $latest = $now; } + my %sch; # maps timestamps to whether they are a scheduled pings + my $i = prevping($earliest); $i = nextping($i); -} + while($i <= $latest) { + $sch{$i} = 1; + $alltimes{$i} = 1; + $i = nextping($i); + } -for my $t (sort(keys(%alltimes))) { - my $missflag = 1; - my @p = (); - for my $l (@ARGV) { - if(defined($th{$l.$t})) { - $missflag = 0; - push(@p, $th{$l.$t}); + for my $t (sort(keys(%alltimes))) { + my $missflag = 1; + my @p = (); + for my $l (@ARGV) { + if(defined($th{$l.$t})) { + $missflag = 0; + push(@p, $th{$l.$t}); + } } + if($sch{$t} && $missflag) { push(@p, annotime('MISSING', $t, 33)); } + if(!$sch{$t}) { push(@p, 'UNSCHED'); } + print $t, " ", join(' + ', @p), "\n"; } - if($sch{$t} && $missflag) { push(@p, annotime('MISSING', $t, 33)); } - if(!$sch{$t}) { push(@p, 'UNSCHED'); } - print $t, " ", join(' + ', @p), "\n"; } + +sub run { + die "USAGE: $0 logfile+\n" if @ARGV < 1; + merge(@ARGV); +} + +run unless caller; + +1; From 8cab7f8c1c10bbc183e24c9c2a307fce16fd5d95 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 17:42:05 -0700 Subject: [PATCH 30/42] Actually merge the tags --- merge.pl | 27 ++++++++++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/merge.pl b/merge.pl index 12da119..bfd7e94 100755 --- a/merge.pl +++ b/merge.pl @@ -34,6 +34,7 @@ BEGIN { require "$ENV{HOME}/.tagtimerc"; } require "util.pl"; +use List::MoreUtils qw(uniq); sub merge { my $e = 0; # number of lines with parse errors @@ -87,18 +88,38 @@ sub merge { $i = nextping($i); } + # We ignore these entries, using them only if we have nothing else + my %ignore = map { $_ => 1 } ('afk off RETRO', 'afk RETRO', 'err'); for my $t (sort(keys(%alltimes))) { my $missflag = 1; my @p = (); + my @backup; for my $l (@ARGV) { if(defined($th{$l.$t})) { $missflag = 0; - push(@p, $th{$l.$t}); + + # Pull out just the tags + my $line = $th{$l.$t}; + $line =~ s/\s+\[.*?\]$//; + my @tags = split(/\s+/, $line); + + if(exists($ignore{$line})) { + # Ignore ignorables, but stash the longest one, + # so we have something in case all entries are ignorable + if($#tags >= $#backup) { @backup = @tags; } + } else { + # Otherwise add our tags to the list + push(@p, @tags); + } } } - if($sch{$t} && $missflag) { push(@p, annotime('MISSING', $t, 33)); } + if($sch{$t} && $missflag) { push(@p, 'MISSING'); } if(!$sch{$t}) { push(@p, 'UNSCHED'); } - print $t, " ", join(' + ', @p), "\n"; + + # If we have tags get the unique set, otherwise use the line we stashed + my @combined = @p ? uniq @p : @backup; + + print $t, ' ', annotime(join(' ', @combined), $t, 72), "\n"; } } From 53a024bc0f9a2df5dc850964e7b0765542a12e1f Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 18:34:11 -0700 Subject: [PATCH 31/42] Replace fill_remote with call to merge --- launch.pl | 46 +++++++++++++++++++--------------------------- merge.pl | 9 ++++++--- 2 files changed, 25 insertions(+), 30 deletions(-) diff --git a/launch.pl b/launch.pl index 8b0c28d..949734c 100755 --- a/launch.pl +++ b/launch.pl @@ -5,6 +5,7 @@ require "$ENV{HOME}/.tagtimerc"; require "${path}util.pl"; +require "${path}merge.pl"; $launchTime = mytime(); @@ -17,22 +18,21 @@ exit(0); } -# figure out the next ping after the last one that's in the log file -$nxtping = parseping($logf); - if(!lockn()) { print "Can't get lock. Exiting.\n" unless $quiet; exit(1); } # Don't wait if we can't get the lock. +# figure out the next ping after the last one that's in the log file +$nxtping = parseping($logf); + if($remote_id ne "" && $nxtping < $launchTime) { # If we have a gap, first try to fill in with stuff from the most recent remote log - fill_remote($nxtping); - - # Re-sync nxtping with any new loglines - $nxtping = parseping($logf); + fill_remote(); } +$nxtping = parseping($logf); + my $editorFlag = 0; print "Filling in RETRO pings ($launchTime <=> $nxtping)\n" unless $quiet; @@ -66,7 +66,7 @@ $verify = nextping(prevping($ts)); # NB: must call prevping before nextping if($ts == $verify) { - fill_remote(nextping($ts)); + fill_remote(); } else { print "Local file has a bad last line:\n$ln"; $nxtping = prevping($launchTime); @@ -150,33 +150,25 @@ sub remoteln { } sub fill_remote { - my ($fill_from) = @_; - print "Backfilling with remote from $fill_from\n" unless $quiet; + print "Downloading remote files...\n" unless $quiet; system("$scp_cmd $remote_log$usr.*.log ."); # Remove our log, we are source of truth for it # Otherwise we overwrite our own edits, bleh unlink "$usr.$remote_id.log"; - $lastremlog = `awk -f get_latest.awk $usr.*.log`; - chomp $lastremlog; + @mergefiles = glob("$usr.*.log"); + + print "Merging pings from remote files " . join(',', @remfiles) . "...\n" unless $quiet; if(-e $logf) { - print "Filling in pings from remote file $lastremlog...\n" unless $quiet; + push(@mergefiles, $logf); system("cp $logf $logf.backup"); - open REMLOG, "<", $lastremlog; - open OURLOG, ">>", $logf; - $numfill = 0; - while() { - /^\s*(\d+)/; # parse out the timestamp for the current line - $curping = $1; - if($curping >= $fill_from) { - print OURLOG $_; - $numfill++; - } - } - print "Filled in $numfill pings\n" unless $quiet; + } + open NEWLOG, ">", "$logf.new"; + if(merge(NEWLOG, @mergefiles) == 0) { + system("mv $logf.merge $logf"); + print "Merge successful\n" unless $quiet; } else { - print "No previous logfile found, using entirety of $lastremlog\n" unless $quiet; - system("cp $lastremlog $logf") + print "Merge errors! Check $logf.merge and resolve errors manually\n"; } } diff --git a/merge.pl b/merge.pl index bfd7e94..1f3e9f9 100755 --- a/merge.pl +++ b/merge.pl @@ -37,6 +37,7 @@ use List::MoreUtils qw(uniq); sub merge { + my $fh = shift; my $e = 0; # number of lines with parse errors my $errstr = ""; # concatenation of bad lines from log files my $earliest = -1; # earliest timestamp in all the log files @@ -74,7 +75,7 @@ sub merge { print "Errors in log file(s): $e. ", "They have to be fixed before this script can run:\n"; print "\n$errstr"; - exit(1); + return 1; } my $now = time(); @@ -119,13 +120,15 @@ sub merge { # If we have tags get the unique set, otherwise use the line we stashed my @combined = @p ? uniq @p : @backup; - print $t, ' ', annotime(join(' ', @combined), $t, 72), "\n"; + print $fh $t, ' ', annotime(join(' ', @combined), $t, 72), "\n"; } + + return 0; } sub run { die "USAGE: $0 logfile+\n" if @ARGV < 1; - merge(@ARGV); + exit(merge(STDOUT, @ARGV)); } run unless caller; From 5995ec15f678b5a37411b74f32b55594c0f8c544 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 18:46:54 -0700 Subject: [PATCH 32/42] Remove calculated values from settings.pl --- launch.pl | 7 +++++++ settings.pl.template | 8 -------- 2 files changed, 7 insertions(+), 8 deletions(-) diff --git a/launch.pl b/launch.pl index 949734c..35c69cf 100755 --- a/launch.pl +++ b/launch.pl @@ -7,6 +7,13 @@ require "${path}util.pl"; require "${path}merge.pl"; +# Generate derived settings used only in this file +$remote_server = "$remote_user\@$remote_host"; +$remote_log = "$remote_server:$remote_path"; +$remote_sshid = $remote_key eq "" ? "" : "-i $remote_key"; +$scp_cmd = "scp $remote_sshid"; +$ssh_cmd = "ssh $remote_sshid"; + $launchTime = mytime(); my $args = join(' ', @ARGV); # supported arguments: test, quiet diff --git a/settings.pl.template b/settings.pl.template index 38b26e5..0a88125 100644 --- a/settings.pl.template +++ b/settings.pl.template @@ -15,14 +15,6 @@ $remote_path = "__REMPATH__"; # CHANGEME to the path on your backup server $remote_id = ""; # CHANGEME to a unique tag for this machine on the backup server # Leaving this blank will disable backup functionality -$remote_server = "$remote_user\@$remote_host"; -$remote_log = "$remote_server:$remote_path"; -$remote_sshid = ""; - if ($remote_key ne "") { $remote_sshid = "-i $remote_key"; } - -$scp_cmd = "scp $remote_sshid"; -$ssh_cmd = "ssh $remote_sshid"; - # If you're using windows, you'll need cygwin and to set this flag to 1: $cygwin = __CYGWIN__; # CHANGEME to 1 if you're using windows/cygwin. From 0799dcc67d2a2d3b5047ccae3ea3f99342ec73e4 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 18:53:22 -0700 Subject: [PATCH 33/42] Make a git commit if the remote is a git repo --- launch.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/launch.pl b/launch.pl index 35c69cf..6855730 100755 --- a/launch.pl +++ b/launch.pl @@ -129,6 +129,7 @@ if($remote_id ne "") { print "Backing up log to remote server...\n" unless $quiet; system("$scp_cmd -C $logf $remote_log$usr.$remote_id.log"); + system("$ssh_cmd $remote_server 'cd $remote_path; [ -d .git ] && git add *.log && git commit -m 'Backup from $remote_id'"); } unlock(); From 8d29e261ce8efe1bc94a74badf0698379f38e977 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 19:36:49 -0700 Subject: [PATCH 34/42] Require util with absolute path Now that we're using it in the daemon --- merge.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merge.pl b/merge.pl index 1f3e9f9..b869191 100755 --- a/merge.pl +++ b/merge.pl @@ -33,7 +33,7 @@ # print t, join('+', @p) BEGIN { require "$ENV{HOME}/.tagtimerc"; } -require "util.pl"; +require "${path}util.pl"; use List::MoreUtils qw(uniq); sub merge { From 66019e1b7f55233c3b074117904db0d3026523be Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Mon, 14 Sep 2020 21:12:30 -0700 Subject: [PATCH 35/42] Log message, commit author --- launch.pl | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/launch.pl b/launch.pl index 6855730..e0fda30 100755 --- a/launch.pl +++ b/launch.pl @@ -129,7 +129,8 @@ if($remote_id ne "") { print "Backing up log to remote server...\n" unless $quiet; system("$scp_cmd -C $logf $remote_log$usr.$remote_id.log"); - system("$ssh_cmd $remote_server 'cd $remote_path; [ -d .git ] && git add *.log && git commit -m 'Backup from $remote_id'"); + print "Making commit if remote is a git repo..."; + system("$ssh_cmd $remote_server 'cd $remote_path; [ -d .git ] && git commit --author=\"Tagtime \" -am \"Backup from $remote_id\"'"); } unlock(); From 4ff2c7c6c16326157038561063e1e9be93519d7e Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sun, 13 Sep 2020 19:30:59 -0700 Subject: [PATCH 36/42] Fixes --- launch.pl | 9 +++++---- merge.pl | 14 +++++++++----- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/launch.pl b/launch.pl index e0fda30..59d1c06 100755 --- a/launch.pl +++ b/launch.pl @@ -165,15 +165,16 @@ sub fill_remote { # Otherwise we overwrite our own edits, bleh unlink "$usr.$remote_id.log"; - @mergefiles = glob("$usr.*.log"); + @mergefiles = glob("$path$usr.*.log"); - print "Merging pings from remote files " . join(',', @remfiles) . "...\n" unless $quiet; + print "Merging pings from remote files...\n" unless $quiet; if(-e $logf) { push(@mergefiles, $logf); system("cp $logf $logf.backup"); } - open NEWLOG, ">", "$logf.new"; - if(merge(NEWLOG, @mergefiles) == 0) { + open NEWLOG, ">", "$logf.merge"; + print(@mergefiles); + if(merge(NEWLOG, 0, @mergefiles) == 0) { system("mv $logf.merge $logf"); print "Merge successful\n" unless $quiet; } else { diff --git a/merge.pl b/merge.pl index b869191..202e57e 100755 --- a/merge.pl +++ b/merge.pl @@ -38,13 +38,15 @@ sub merge { my $fh = shift; + my $fill_now = shift; my $e = 0; # number of lines with parse errors my $errstr = ""; # concatenation of bad lines from log files my $earliest = -1; # earliest timestamp in all the log files my $latest = 0; # latest timestamp in all the log files my %th; # maps logfile+timestamp to tags for that log for that ping my %alltimes; # maps all timestamps to 1 - for my $logfile (@_) { + my @files = @_; + for my $logfile (@files) { open(LOG, $logfile) or die; $prevts = 0; # remember the previous timestamp while($line = ) { @@ -78,8 +80,10 @@ sub merge { return 1; } - my $now = time(); - if($now > $latest) { $latest = $now; } + if($fill_now) { + my $now = time(); + if($now > $latest) { $latest = $now; } + } my %sch; # maps timestamps to whether they are a scheduled pings my $i = prevping($earliest); $i = nextping($i); @@ -95,7 +99,7 @@ sub merge { my $missflag = 1; my @p = (); my @backup; - for my $l (@ARGV) { + for my $l (@files) { if(defined($th{$l.$t})) { $missflag = 0; @@ -128,7 +132,7 @@ sub merge { sub run { die "USAGE: $0 logfile+\n" if @ARGV < 1; - exit(merge(STDOUT, @ARGV)); + exit(merge(STDOUT, 1, @ARGV)); } run unless caller; From 2de88d3236b4c93ef51c14402bcdd24bd1b5198a Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Mon, 14 Sep 2020 21:26:00 -0700 Subject: [PATCH 37/42] Add filename for NON-MONOTONE --- merge.pl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/merge.pl b/merge.pl index 202e57e..f6de151 100755 --- a/merge.pl +++ b/merge.pl @@ -59,7 +59,7 @@ sub merge { my $ts = shift(@tags); if($ts <= $prevts) { $e++; - $errstr .= "NON-MONOTONE:\n$line"; + $errstr .= "NON-MONOTONE in $logfile:\n$line"; next; } $prevts = $ts; From 569c6be148b9ab8f675fe1c3764bad09930b6059 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Thu, 17 Sep 2020 19:07:57 -0700 Subject: [PATCH 38/42] Add debug function that forces a flush In case we exit/are killed first...this shouldn't be necessary, but I seem to be dropping log messages, so... --- launch.pl | 28 ++++++++++++++-------------- util.pl | 7 +++++++ 2 files changed, 21 insertions(+), 14 deletions(-) diff --git a/launch.pl b/launch.pl index 59d1c06..c2565f9 100755 --- a/launch.pl +++ b/launch.pl @@ -26,8 +26,8 @@ } if(!lockn()) { - print "Can't get lock. Exiting.\n" unless $quiet; - exit(1); + debug("Can't get lock. Exiting."); + exit(1); } # Don't wait if we can't get the lock. # figure out the next ping after the last one that's in the log file @@ -42,7 +42,7 @@ my $editorFlag = 0; -print "Filling in RETRO pings ($launchTime <=> $nxtping)\n" unless $quiet; +debug("Filling in RETRO pings ($launchTime <=> $nxtping)"); # First, if we missed any pings by more than $retrothresh seconds for no # apparent reason, then assume the computer was off and auto-log them. while($nxtping < $launchTime-$retrothresh) { @@ -62,14 +62,14 @@ } my($ts,$ln) = lastln(); - print "Processing ping response ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + debug("Processing ping response ($ts <=> $nxtping, ef=$editorFlag)"); # First, check to see if we have remote pings to fill in, if this computer # was just sitting with a ping window up while they were being answered elsewhere if($ts != $nxtping) { my ($rts,$rln) = remoteln(); if ($rts > $ts) { - print "$rts > $ts, filling from remote\n" unless $quiet; + debug("$rts > $ts, filling from remote"); $verify = nextping(prevping($ts)); # NB: must call prevping before nextping if($ts == $verify) { @@ -83,17 +83,17 @@ $verify = nextping(prevping($ts)); if($ts == $verify) { - print "New last timestamp: $ts\n" unless $quiet; + debug("New last timestamp: $ts"); } else { print "Remote file has a bad last line:\n$ln"; $nxtping = prevping($launchTime); } } else { - print "$rts <= $ts, nothing to fill from remote\n" unless $quiet; + debug("$rts <= $ts, nothing to fill from remote"); } } - print "Checked from remote ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + debug("Checked from remote ($ts <=> $nxtping, ef=$editorFlag)"); if($ts != $nxtping) { # in case, eg, we closed the window w/o answering. # suppose there's a ping window waiting (call it ping 1), and while it's @@ -112,7 +112,7 @@ $editorFlag = 1; } - print "Generated err pings ($ts <=> $nxtping, ef=$editorFlag)\n" unless $quiet; + debug("Generated err pings ($ts <=> $nxtping, ef=$editorFlag)"); $lstping = $nxtping; $nxtping = nextping($nxtping); # Here's where we would add an artificial gap of $nxtping-$lstping. @@ -127,9 +127,9 @@ } while($nxtping <= mytime()); if($remote_id ne "") { - print "Backing up log to remote server...\n" unless $quiet; + debug("Backing up log to remote server..."); system("$scp_cmd -C $logf $remote_log$usr.$remote_id.log"); - print "Making commit if remote is a git repo..."; + debug("Making commit if remote is a git repo..."); system("$ssh_cmd $remote_server 'cd $remote_path; [ -d .git ] && git commit --author=\"Tagtime \" -am \"Backup from $remote_id\"'"); } unlock(); @@ -159,7 +159,7 @@ sub remoteln { } sub fill_remote { - print "Downloading remote files...\n" unless $quiet; + debug("Downloading remote files..."); system("$scp_cmd $remote_log$usr.*.log ."); # Remove our log, we are source of truth for it # Otherwise we overwrite our own edits, bleh @@ -167,7 +167,7 @@ sub fill_remote { @mergefiles = glob("$path$usr.*.log"); - print "Merging pings from remote files...\n" unless $quiet; + debug("Merging pings from remote files..."); if(-e $logf) { push(@mergefiles, $logf); system("cp $logf $logf.backup"); @@ -176,7 +176,7 @@ sub fill_remote { print(@mergefiles); if(merge(NEWLOG, 0, @mergefiles) == 0) { system("mv $logf.merge $logf"); - print "Merge successful\n" unless $quiet; + debug("Merge successful"); } else { print "Merge errors! Check $logf.merge and resolve errors manually\n"; } diff --git a/util.pl b/util.pl index f1b6c78..4917f1c 100644 --- a/util.pl +++ b/util.pl @@ -237,6 +237,13 @@ sub slog { close(F); } +sub debug { + my $line = shift; + unless($quiet) { + print "$line\n"; $| = 1; + } +} + # double-digit: takes number from 0-99, returns 2-char string eg "03" or "42". sub dd { my($n) = @_; return padl($n, "0", 2); } # simpler but less general version: return ($n<=9 && $n>=0 ? "0".$n : $n) From f1dfbdc57c2974bc8498e0ca345ec1453ec010e3 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Thu, 17 Sep 2020 19:13:21 -0700 Subject: [PATCH 39/42] Pop up an editor for merge errors This shouldn't be a silent failure --- launch.pl | 2 +- merge.pl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/launch.pl b/launch.pl index c2565f9..9a481b4 100755 --- a/launch.pl +++ b/launch.pl @@ -178,7 +178,7 @@ sub fill_remote { system("mv $logf.merge $logf"); debug("Merge successful"); } else { - print "Merge errors! Check $logf.merge and resolve errors manually\n"; + editor("$logf.merge", "Merge errors! Please resolve errors manually") } } diff --git a/merge.pl b/merge.pl index f6de151..9988603 100755 --- a/merge.pl +++ b/merge.pl @@ -75,7 +75,7 @@ sub merge { if($e>0) { print "Errors in log file(s): $e. ", - "They have to be fixed before this script can run:\n"; + "Please fix errors and then re-run merge.pl manually:\n"; print "\n$errstr"; return 1; } From 53f8105708cfb1afe6487c7544143880c73f75ed Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Thu, 17 Sep 2020 21:53:46 -0700 Subject: [PATCH 40/42] Update $nxtping when we pull from remote This should fix the erroneous err bug --- launch.pl | 1 + 1 file changed, 1 insertion(+) diff --git a/launch.pl b/launch.pl index 9a481b4..8f8fa4e 100755 --- a/launch.pl +++ b/launch.pl @@ -84,6 +84,7 @@ $verify = nextping(prevping($ts)); if($ts == $verify) { debug("New last timestamp: $ts"); + $nxtping = $ts } else { print "Remote file has a bad last line:\n$ln"; $nxtping = prevping($launchTime); From 05a4ed05a6e645c82797c506c73210b3c3200282 Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 3 Apr 2021 10:29:17 -0700 Subject: [PATCH 41/42] Get more aggressive about failed merges The previous warning strategy was not effective because a) the user can't do anything from an editor to resolve this and b) the error messages weren't going to the file anyway, so it showed an empty file So now we just check for the presence of a merge file (if all goes well it will be cleaned up afterward) and throw up a big ol' warning if it exists --- launch.pl | 2 +- merge.pl | 6 +++--- ping.pl | 20 ++++++++++++++++++++ 3 files changed, 24 insertions(+), 4 deletions(-) diff --git a/launch.pl b/launch.pl index 8f8fa4e..70c51e9 100755 --- a/launch.pl +++ b/launch.pl @@ -179,7 +179,7 @@ sub fill_remote { system("mv $logf.merge $logf"); debug("Merge successful"); } else { - editor("$logf.merge", "Merge errors! Please resolve errors manually") + debug("Merge errors! Leaving empty merge file in place to signal ping") } } diff --git a/merge.pl b/merge.pl index 9988603..1f19096 100755 --- a/merge.pl +++ b/merge.pl @@ -47,7 +47,7 @@ sub merge { my %alltimes; # maps all timestamps to 1 my @files = @_; for my $logfile (@files) { - open(LOG, $logfile) or die; + open(LOG, $logfile) or die "Couldn't open $logfile"; $prevts = 0; # remember the previous timestamp while($line = ) { if(!parsable($line)) { @@ -74,9 +74,9 @@ sub merge { } if($e>0) { - print "Errors in log file(s): $e. ", + print STDERR "Errors in log file(s): $e. ", "Please fix errors and then re-run merge.pl manually:\n"; - print "\n$errstr"; + print STDERR "\n$errstr"; return 1; } diff --git a/ping.pl b/ping.pl index 7bcc6f4..7744320 100755 --- a/ping.pl +++ b/ping.pl @@ -35,6 +35,26 @@ # instances are invoked, but launch.pl will only launch one at a time. #lockb(); # wait till we can get the lock. +# Warn about merge errors if there's a merge file remaining +if(-e "$logf.merge") { + print divider(""), "\n"; + print divider(" WARNING "x8), "\n"; + print divider(""), "\n"; + print "Failed merge to resolve!\n"; + print < $usr.log.merge + +When you are satisfied, replace your logfile with the merged file, removing the +merge file. Your remote logs will not get synced until you do so! +EOS + print divider(""), "\n\n"; +} + if($pingTime-$t > 9) { print divider(""), "\n"; print divider(" WARNING "x8), "\n"; From 3678174dccd069285bdadd32376374a72b871c1e Mon Sep 17 00:00:00 2001 From: Joel Bradshaw Date: Sat, 24 Apr 2021 12:32:55 -0700 Subject: [PATCH 42/42] Give list of files with errors in merge --- merge.pl | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/merge.pl b/merge.pl index 1f19096..b3b4920 100755 --- a/merge.pl +++ b/merge.pl @@ -41,6 +41,7 @@ sub merge { my $fill_now = shift; my $e = 0; # number of lines with parse errors my $errstr = ""; # concatenation of bad lines from log files + my %errfiles; # map of files with errors my $earliest = -1; # earliest timestamp in all the log files my $latest = 0; # latest timestamp in all the log files my %th; # maps logfile+timestamp to tags for that log for that ping @@ -53,6 +54,7 @@ sub merge { if(!parsable($line)) { $e++; $errstr .= $line; + $errfiles{$logfile} = $e; next; } my @tags = split(/\s+/, $line); @@ -60,6 +62,7 @@ sub merge { if($ts <= $prevts) { $e++; $errstr .= "NON-MONOTONE in $logfile:\n$line"; + $errfiles{$logfile} = $e; next; } $prevts = $ts; @@ -74,7 +77,8 @@ sub merge { } if($e>0) { - print STDERR "Errors in log file(s): $e. ", + my $files = join ",", keys %errfiles; + print STDERR "Errors in log file(s): $files ", "Please fix errors and then re-run merge.pl manually:\n"; print STDERR "\n$errstr"; return 1;