From a92419df9dccd87c15f4fef24084725ec6c10b6f Mon Sep 17 00:00:00 2001 From: wbudic Date: Fri, 13 Aug 2021 17:50:12 +1000 Subject: [PATCH] Reworked , cross db engine backup/restore. --- Current Development Check List.md | 2 +- htdocs/cgi-bin/config.cgi | 358 +++++++++++++++++------------- htdocs/cgi-bin/main.cgi | 2 +- 3 files changed, 207 insertions(+), 155 deletions(-) diff --git a/Current Development Check List.md b/Current Development Check List.md index c7a9382..aa9a4af 100644 --- a/Current Development Check List.md +++ b/Current Development Check List.md @@ -6,7 +6,7 @@ ### New Development -* Implement backup/restore on Pg based data. Restore only partially works from older backup. +* ✔ Implement backup/restore on Pg based data. Restore only partially works from older backup. * ✔ Bug 35. SQL migration, version update, not working for PG based databases. * ✔ JS - Event, on expense sum in log numbers found at beginning of lines. * ✔ Update to CNF v.2.2, branch to $RELEASE_VER = 2.3, Earth Stage initial. diff --git a/htdocs/cgi-bin/config.cgi b/htdocs/cgi-bin/config.cgi index c9680a1..0732b69 100755 --- a/htdocs/cgi-bin/config.cgi +++ b/htdocs/cgi-bin/config.cgi @@ -8,8 +8,11 @@ use strict; use warnings; use DBI; +use DBD::Pg; +use DBD::Pg qw(:pg_types); use Exception::Class ('LifeLogException'); use Syntax::Keyword::Try; +use CGI; use DateTime::Format::SQLite; use Date::Language; @@ -21,10 +24,10 @@ use Sys::Syslog qw(:DEFAULT :standard :macros); #openLog, closelog macros #DEFAULT SETTINGS HERE! use lib "system/modules"; require Settings; -## - #15mg data post limit $CGI::POST_MAX = 1024 * 15000; +## + my ($RDS,$TR_STATUS,$LOGOUT,$ERROR) = ("","",0,""); my $sys = `uname -n`; my $db = Settings::fetchDBSettings(); @@ -380,7 +383,6 @@ if((Settings::anon("backup_enabled") == 0)){ # # Page printout from here! # - print qq(
$frmVars
@@ -395,7 +397,7 @@ print qq(
- + @@ -404,29 +406,28 @@ print qq( $inpRestore
- - - - - - - + + + + + + - - - +

Backup File Format

$bck_list

CSV File Format

Notice: (Obsolete feature, use not recommended!)
- Import Categories:
-

CSV File Format

Notice: (This is an obsolete feature, use is not recommended!)

This Servers DSN is ->
).Settings::dsn().qq(

+ Import Categories: +
+
Export Categories:  
- Notice: (Obsolete feature, use not recommended!)
+
+
Import Log:
+
Export Log:$inpCVS -

For Server -> $sys -> $dbname


↟ Go to Top of page
@@ -469,60 +470,9 @@ print qq(

- -
-
-

L-Tags Specs

-

- Life Log Tags are simple markup allowing fancy formatting and functionality - for your logs HTML layout. -

-

- <<B<{Text To Bold}>> -

-

- <<I<{Text To Italic}>> -

-

- <<TITLE<{Title Text}>> -

-

- <<LIST<{List of items delimited by new line to terminate item or with '~' otherwise.}>> -

-

- <<IMG<{url to image}>> -

-

- <<FRM<{file name}_frm.png}>>

- *_frm.png images file pairs are located in the ./images folder of the cgi-bin directory.
- These are manually resized by the user. Next to the original. - Otherwise considered as stand alone icons. *_frm.png Image resized to -> width="210" height="120" -
Example: - 

-        ../cgi-bin/images/
-            my_cat_simon_frm.png
-            my_cat_simon.jpg
-
-          For log entry, place:
-
-      <<FRM>my_cat_simon_frm.png> <<TITLE<Simon The Cat>>
-      This is my pet, can you hold him for a week while I am on holiday?
-
-

- -

<<LNK<{url to image}>>

-

- Explicitly tag an URL in the log entry. - Required if using in log IMG or FRM tags. - Otherwise link appears as plain text. -

-
-

Log Page Particulars

- ↟ or ↡ - Jump links to top or bottom of page respectivelly. -
↟ -
+ -
+

Back to Main Log

LOGOUT

@@ -963,25 +913,6 @@ sub renumerate { } - - - - - - - - - - - - - - - - - - - sub resetCategories { $db->do("DELETE FROM CAT;"); $db->do("DROP TABLE CAT;"); @@ -1122,51 +1053,69 @@ try{ exit; }; } -sub backup { - my $pass = Settings::pass(); - my $ball = 'bck_'.$today->strftime('%Y%m%d%H%M%S_')."_$dbname.osz"; - my $pipe = "tar czf - ".Settings::logPath().'main.cnf' ." ". Settings::dbFile()." | openssl enc -e -des-ede3-cfb -salt -S ".Settings->CIPHER_KEY." -pass pass:$pass-$alias -out ".Settings::logPath().$ball." 2>/dev/null"; - my $rez = `$pipe`; - print $cgi->header(-charset=>"UTF-8", -type=>"application/octet-stream", -attachment=>$ball); - open (my $TAR, "<", Settings::logPath().$ball) or die "Failed creating backup -> $ball"; - while(<$TAR>){print $_;} - close $TAR; - exit; -} - -package DBMigStats { - - sub new { - my $class = shift; - my $self = bless {cats_ins => 0, cats_upd => 0, logs_ins => 0, logs_upd => 0, notes => 0}, $class; - } +sub backup { + my $pass = Settings::pass(); + my @dr = split(':', Settings::dbSrc()); + my $ball = 'bck_'.$today->strftime('%Y%m%d%H%M%S_').$dr[1]."_$dbname.osz"; + + my $file = Settings::logPath().'data_'.$dr[1].'_'."$dbname"."_log.db"; + my $dsn= "DBI:SQLite:dbname=$file"; + my $weProgress = Settings::isProgressDB(); + if($weProgress){ + try{$pass = uc crypt $pass, hex Settings->CIPHER_KEY; + unlink $file if -e $file; # we will recreate it next. + }catch{}; + my $dbB = DBI->connect($dsn, $alias, $pass, { AutoCommit => 1, RaiseError => 1 }) or + LifeLogException->throw(error=>"Invalid database! $dsn [$@]", show_trace=>&Settings::debug); + &Settings::resetToDefaultDriver; + $dbB->do(&Settings::createCATStmt); + $dbB->do(&Settings::createLOGStmt); + $dbB->do(&Settings::createNOTEStmt); + + my $in = $dbB->prepare('INSERT INTO CAT VALUES (?,?,?)'); + my $st = Settings::selectRecords($db,'SELECT * FROM CAT;'); + while(my @c = $st->fetchrow_array()){ + $in->execute($c[0],$c[1],$c[2]); + } - sub cats_inserts(){my $s = shift;return $s->{cats_ins}} - sub cats_inserts_incr() {my $s = shift; $s->{cats_ins}++} - sub cats_updates(){my $s = shift;return $s->{cats_upd}} - sub cats_updates_incr() {my $s = shift; $s->{cats_upd}++} - - sub logs_inserts(){my $s = shift;return $s->{logs_ins}} - sub logs_inserts_incr(){my $s = shift; $s->{logs_ins}++} - sub logs_updates(){my $s = shift;return $s->{logs_upd}} - sub logs_updates_incr(){my $s = shift; $s->{logs_upd}++} - - sub notes() {my $s = shift;return $s->{notes}} - sub notes_incr() {my $s = shift; $s->{notes}++} + $in = $dbB->prepare('INSERT INTO LOG (ID_CAT, DATE, LOG, RTF, AMOUNT, AFLAG, STICKY) VALUES (?,?,?,?,?,?,?);'); + $st = Settings::selectRecords($db,'SELECT ID_CAT, DATE, LOG, RTF, AMOUNT, AFLAG, STICKY FROM LOG order by DATE;'); + while(my @c = $st->fetchrow_array()){ + $in->execute($c[0],$c[1],$c[2],$c[3],$c[4],$c[5],$c[6]); + } + $in = $dbB->prepare('INSERT INTO NOTES VALUES (?,?)'); + $st = Settings::selectRecords($db,'SELECT LID, DOC FROM NOTES;'); + while(my @c = $st->fetchrow_array()){ + $in->execute($c[0],$c[1]); + } + $dbB->disconnect(); + }else{ + $file = Settings::dbFile(); + } + print $cgi->header(-charset=>"UTF-8", -type=>"application/octet-stream", -attachment=>$ball); + my $pipe = "tar czf - ".Settings::logPath().'main.cnf' ." ".$file." | openssl enc -e -des-ede3-cfb -salt -S ". + Settings->CIPHER_KEY." -pass pass:$pass-$alias -out ".Settings::logPath().$ball." 2>/dev/null"; + my $rez = `$pipe`; + open (my $TAR, "<", Settings::logPath().$ball) or die "Failed creating backup -> $ball"; + while(<$TAR>){print $_;} + close $TAR; + unlink $file if $weProgress; + exit; } sub restore { my $file = shift; - my ($tar,$pipe,@br,$stdout,$b_db); + my ($tar,$pipe,@br,$stdout, $b_db); my $pass = Settings::pass(); my $hndl = $cgi->param('data_bck'); my $dbck = &Settings::logPath."bck/"; `mkdir $dbck` if (!-d $dbck); + my $stage = "Initial"; - try{ +try{ getHeader(); print $cgi->start_html; @@ -1184,37 +1133,63 @@ my $stdout = capture_stdout { print "
Reading-> $hndl
"; $tar = $dbck.$hndl; } - $tar =~ s/osz$/tar/; + + + my $srcIsPg = 0; my $pipe; - open ($pipe, "| openssl enc -d -des-ede3-cfb -salt -S ".Settings->CIPHER_KEY." -pass pass:$pass-$alias -in /dev/stdin 2>/dev/null > $tar"); + my $passw = $pass; $passw = uc crypt $pass, hex Settings->CIPHER_KEY if &Settings::isProgressDB; + open ($pipe, "| openssl enc -d -des-ede3-cfb -salt -S ". + Settings->CIPHER_KEY." -pass pass:$passw-$alias -in /dev/stdin 2>/dev/null > $tar"); while(<$hndl>){print $pipe $_;}; close $pipe; close $hndl; print "
\n";
         
 
-        my $m1 = "it is not permitted to restore another aliases log backup.";
+        my $m1 = "it is not permitted to restore from anothers backup file.";
         $m1= "has your log password changed?" if ($tar=~/_data_$alias/);
+        $stage = "Extraction";
 
         my $cmd = `tar tvf $tar 2>/dev/null` 
-         or die qq(, possible an security issue, $m1\nBACKUP FILE INVALID! $tar\nYour data alias is: $alias\nYour LifeLog version is:), Settings::release()."\n";
-
+         or die qq(, Error: A possible security issue, $m1\n
 BACKUP FILE HAS BEEN INVALIDATED!
+          $tar\nYour alias is: $alias:$passw\n

+          Your DSN  is: ).Settings::dsn().qq(

+          Your LifeLog version is:), Settings::release()."\n";
+        
         print "Contents->\n".$cmd."\n\n";
         $cmd = `tar xzvf $tar -C $dbck --strip-components 1 2>/dev/null` or die "Failed extracting $tar";
         print "Extracted->\n".$cmd."\n" or die "Failed extracting $tar";;
-
-        my $b_base = $dbck.'data_'.$dbname.'_log.db'; 
+        my @dr = split(':', Settings::dbSrc());
+        my $b_base = $dbck.'data_'.$dbname.'_log.db';  
+        
+        # We check if db file has been extracted first?
+        unless(-e $b_base){                  
+                  if (&Settings::isProgressDB){
+                      $b_base = $dbck.'data_'.$dr[1].'_'.$dbname.'_log.db'
+                  }else{ # maybe the source is a Pg db backup?
+                      $b_base = $dbck.'data_Pg_'.$dbname.'_log.db';
+                      $srcIsPg = 1;
+                  }
+                  unless(-e $b_base){
+                      die "Failed to locate database in archive -> $b_base";
+                  }
+        }
         my $dsn= "DBI:SQLite:dbname=$b_base";
-        $b_db = DBI->connect($dsn, $alias, $pass, { RaiseError => 1 }) or LifeLogException->throw(error=>"Invalid database! $dsn->$hndl [$@]", show_trace=>&Settings::debug);
-        print "Connected to -> $dsn\n";
 
+        $b_db = DBI->connect($dsn, $alias, $pass, { RaiseError => 1 }) or 
+                 LifeLogException->throw(error=>"Invalid database! $dsn->$hndl [$@]", show_trace=>&Settings::debug);
+
+        print "Connected to -> ".Settings::dsn()."\n";
+        $stage = "Merging categories table.";
         print "Merging from backup categories table...";
         my $stats =  DBMigStats -> new();
         my $insCAT   = $db->prepare('INSERT INTO CAT (ID, NAME, DESCRIPTION) VALUES(?,?,?);') or die "Failed CAT prepare.";
         my $b_pst = Settings::selectRecords($b_db,'SELECT ID, NAME, DESCRIPTION FROM CAT;');
         
         while ( @br = $b_pst->fetchrow_array() ) {
+            next if not $br[0]; #@2021-08-12 For some reason this still could be null 
+            $stage .= "
id:".$br[0]."->".$br[1];
             my $pst = Settings::selectRecords($db, "SELECT ID, NAME, DESCRIPTION FROM CAT WHERE ID=".$br[0].";");
             my @ext = $pst->fetchrow_array();
             if(scalar(@ext)==0){
@@ -1231,56 +1206,110 @@ my $stdout = capture_stdout {
         print "\nFinished with merging CAT table.\n";
         print "There where -> ". $stats->cats_inserts(). " inserts, and ". $stats->cats_updates(). " updates.\n";
 
+        $stage = "Merging backup LOG";
         print "\n\nMerging from backup LOG table...\n";
+
         my %backupLIDS =();
+        my $CI = 'rowid'; $CI = 'ID' if Settings::isProgressDB();
         my $insLOG   = $db->prepare('INSERT INTO LOG (ID_CAT, DATE, LOG, RTF, AMOUNT, AFLAG, STICKY) VALUES(?,?,?,?,?,?,?);')or die "Failed LOG prepare.";
 
-        $b_pst = Settings::selectRecords($b_db,'SELECT ID, ID_CAT, DATE, LOG, RTF, AMOUNT, AFLAG, STICKY FROM '.Settings->VW_LOG);
+        $b_pst = Settings::selectRecords($b_db,"SELECT rowid, ID_CAT, DATE, LOG, RTF, AMOUNT, AFLAG, STICKY FROM LOG;");
         while ( @br = $b_pst->fetchrow_array() ) {
-            my $pst = Settings::selectRecords($db,"SELECT DATE FROM ".Settings->VW_LOG." WHERE DATE='".$br[2]."';");
+            my $dt  = $br[2];
+            my $pst = Settings::selectRecords($db,"SELECT DATE FROM LOG WHERE DATE='".$dt."';");
             my @ext = $pst->fetchrow_array();
             if(scalar(@ext)==0){
-                $insLOG->execute($br[1],$br[2],$br[3],$br[4],$br[5],$br[6],$br[7]);
-                print "Added->".$br[0]."|".$br[2]."|".$br[3]."\n"; $stats->logs_inserts_incr();
-                if($br[4]){                    
-                    $pst = Settings::selectRecords($db, "SELECT max(id) FROM ".Settings->VW_LOG);
-                    my @r = $pst->fetchrow_array();
-                    $backupLIDS{$br[0]} = $r[0];
+                try{
+                            $insLOG->execute($br[1],$br[2],$br[3],$br[4],$br[5],$br[6],$br[7]);
+                            print "Added->".$br[0]."|".$br[2]."|".$br[3]."\n"; $stats->logs_inserts_incr();
+                            if($br[4]){                    
+                                $pst = Settings::selectRecords($db, "SELECT max($CI) FROM LOG");
+                                my @r = $pst->fetchrow_array();
+                                $backupLIDS{$br[0]} = $r[0];
+                            }
+                }
+                catch{
+                    print "Insert insert of ->[".$br[0]."|".$br[2]."|\n".$br[3]."]
Error -> $@\n";
                 }
             }
         }
         print "\nFinished with merging LOG table.\n";
         print "There where -> ". $stats->logs_inserts(). " inserts.\n";
 
+        $stage = "Merging Notes";
         print "\nMerging from backup NOTES table...\n";
         my $insNOTES   = $db->prepare('INSERT INTO NOTES (LID, DOC) VALUES(?,?);') or die "Failed NOTES prepare.";
         $b_pst = Settings::selectRecords($b_db,'SELECT LID, DOC FROM NOTES;');
         while ( @br = $b_pst->fetchrow_array() ) {
             my $in_id = $backupLIDS{$br[0]};
             if($in_id && $br[1]){
-               $insNOTES->execute($in_id, $br[1]) or die "Failed NOTES INSERT[".$br[0]."]";
-               print "Added NOTES -> LID:$in_id\n";                
+                if(Settings::isProgressDB()){                    
+                    $insNOTES->bind_param(1, $in_id);                    
+                    try{                    
+                        use IO::Compress::Gzip qw(gzip $GzipError);
+                        use Compress::Zlib;
+                        use Crypt::Blowfish;
+                        use Crypt::CBC;
+                        sub cryptKey {
+                            my $p = shift;
+                            my $r = $alias.$p.Settings->CIPHER_KEY;
+                            $r    =~ s/(.)/sprintf '%04x', ord $1/seg;        
+                        return  substr $r.Settings->CIPHER_PADDING, 0, 58;
+                        }
+                        if( not $srcIsPg ){ #IT is NOT PG BCK to PG DB
+                            # With Pg the password we don't encrypt itself, so we need to redo the binary :(.
+                            my $d = uncompress($br[1]);
+                            my $cipher = Crypt::CBC->new(-key  => cryptKey($passw), -cipher => 'Blowfish');
+                            my $doc = $cipher->decrypt($d);
+                            #print $doc;
+                            $cipher = Crypt::CBC->new(-key  => cryptKey($pass), -cipher => 'Blowfish');
+                            $doc = compress($cipher->encrypt($doc));
+                            $insNOTES->bind_param(2, $doc, { pg_type => DBD::Pg::PG_BYTEA });
+                            $insNOTES->execute() or die "Failed NOTES INSERT[".$br[0]."]";
+                        }else{
+                            $insNOTES->bind_param(2, $br[1], { pg_type => DBD::Pg::PG_BYTEA });
+                            $insNOTES->execute() or die "Failed NOTES INSERT[".$br[0]."]";
+                        }
+                        print "Added ".$dr[1]." NOTES -> LID:$in_id\n";
+
+
+                    }
+                    catch{
+                        print "FAILED TO INSERT NOTES -> LID:$in_id Err:$@\n";
+                    } 
+                    
+                }else{
+                    try{
+                        $insNOTES->execute($in_id, $br[1]);
+                        print "Added NOTES -> LID:$in_id\n";
+                    }
+                    catch{
+                        print "FAILED TO INSERT NOTES -> LID:$in_id Err:$@\n";
+                    } 
+                }                   
             }
         }
         print "\nFinished with merging NOTES table.\n";
         print "Note that the merge didn't recover documents for any existing log entries.\n";
-        print "To do this, delete those log entries, then run restore again.\n";
-        #`rm -rf $dbck/`;
+        print "To do this, delete those log entries, then run restore again.\n";        
         print "Done!\n";
         print "Restore ended: ".Settings::today(), "\n";
 };      print $stdout;
 
-        my $fh; open( $fh, ">>", Settings::logPath()."backup_restore.log");
-                print $fh $stdout;
-                close $fh;
+my $fh; open( $fh, ">>", Settings::logPath()."backup_restore.log");
+        print $fh $stdout;
+        close $fh;
 
-        $b_db->disconnect();
-        $db->disconnect();       
-    }
-    catch{
-        $ERROR = "Restore Failed!hndl->$hndl $@ \n";
-        $ERROR .= "br:[@br]" if(@br);
-    };
+$b_db->disconnect();
+$db->disconnect();
+`rm -rf $dbck/`;
+
+}
+catch{
+    $ERROR = "
Full Restore Failed!
hndl->$hndl 
$@ \n";
+    $ERROR .= "br:[@br]" if(@br);
+    $ERROR .= "
Failed at stage: $stage";
+};
 
     my $back = $cgi->url( -relative => 1 );
     print $ERROR if($ERROR);
@@ -1291,6 +1320,29 @@ my $stdout = capture_stdout {
 
 }
 
+package DBMigStats {
+
+    
+    sub new {
+        my $class = shift;
+        my $self = bless {cats_ins  => 0, cats_upd  => 0, logs_ins  => 0, logs_upd  => 0, notes => 0}, $class;
+    }
+
+    sub cats_inserts(){my $s = shift;return $s->{cats_ins}}
+    sub cats_inserts_incr() {my $s = shift; $s->{cats_ins}++}
+    sub cats_updates(){my $s = shift;return $s->{cats_upd}}
+    sub cats_updates_incr() {my $s = shift; $s->{cats_upd}++}
+    
+    sub logs_inserts(){my $s = shift;return $s->{logs_ins}}
+    sub logs_inserts_incr(){my $s = shift;  $s->{logs_ins}++}
+    sub logs_updates(){my $s = shift;return $s->{logs_upd}}
+    sub logs_updates_incr(){my $s = shift;  $s->{logs_upd}++}
+    
+    sub notes()      {my $s = shift;return  $s->{notes}}
+    sub notes_incr() {my $s = shift; $s->{notes}++}
+
+}
+
 sub exportToCSV {
     try{
         my $csv = Text::CSV->new ( { binary => 1, strict => 1,eol => $/ } );
diff --git a/htdocs/cgi-bin/main.cgi b/htdocs/cgi-bin/main.cgi
index fd78f38..c085461 100755
--- a/htdocs/cgi-bin/main.cgi
+++ b/htdocs/cgi-bin/main.cgi
@@ -1187,7 +1187,7 @@ try {
                       toBuf("\nINSERT INTO NOTES($lids[0], {DOC[$date]})") if ($DEBUG);
                       $st = $db->prepare("INSERT INTO NOTES(LID, DOC) VALUES (?, ?);");
                       if(Settings::isProgressDB()){
-                          $st->bind_param(1, $lids[0]);                            
+                          $st->bind_param(1, $lids[0]);           
                           $st->bind_param(2, $gzero[0],{ pg_type => DBD::Pg::PG_BYTEA });
                           $st->execute();
                       }else{
-- 
2.34.1