This patch fixes many problems in the handling of GPDB CSV format logs for the frmSta...
authordpage <dpage@a7884b65-44f6-0310-8a51-81a127f17b15>
Tue, 9 Jun 2009 11:19:10 +0000 (11:19 +0000)
committerdpage <dpage@a7884b65-44f6-0310-8a51-81a127f17b15>
Tue, 9 Jun 2009 11:19:10 +0000 (11:19 +0000)
It's also a useful start for supporting PostgreSQL 8.4 CSV format logs.
[Chuck McDevitt]

git-svn-id: svn://svn.pgadmin.org/trunk/pgadmin3@7901 a7884b65-44f6-0310-8a51-81a127f17b15

pgadmin/frm/frmMainConfig.cpp
pgadmin/frm/frmStatus.cpp
pgadmin/include/frm/frmStatus.h
pgadmin/include/precomp.h
pgadmin/include/utils/csvfiles.h [new file with mode: 0644]
pgadmin/include/utils/module.mk
pgadmin/pgAdmin3.vcproj
pgadmin/utils/csvfiles.cpp [new file with mode: 0644]
pgadmin/utils/module.mk

index 0f29f95174aaa18bfcecade5f46ea51bc948eec8..4dec82f3f97c8085281128a657665e6e5fdc86b6 100644 (file)
@@ -298,7 +298,7 @@ void frmMainConfig::WriteFile(pgConn *conn)
     {
         pgSettingItem *item = options[cfgList->GetText(i)];
 
-        if (item && item->newLine && !item->orgLine)
+        if (item && item->newLine && item->newLine->item && !item->orgLine)
             str.Append(item->newLine->GetNewText() + wxT("\n"));
     }
 
index 0d5a8303043528dd52facbbefdb4a303ba72fc1e..d93e4635bb407ff6923fb9c1e36f30f40de2926e 100644 (file)
@@ -29,6 +29,7 @@
 #include "utils/pgfeatures.h"
 #include "schema/pgServer.h"
 #include "ctl/ctlMenuToolbar.h"
+#include "utils/csvfiles.h"
 
 // Icons
 #include "images/clip_copy.xpm"
@@ -282,6 +283,13 @@ frmStatus::frmStatus(frmMain *form, const wxString& _title, pgConn *conn) : pgFr
     {
         manager.GetPane(wxT("Transactions")).Show(false);
     }
+    else if (connection->GetIsGreenplum())
+    {
+        // GPDB doesn't have external global transactions.
+        // Perhaps we should use this display to show our
+        // global xid to local xid mappings?
+        manager.GetPane(wxT("Transactions")).Show(false);
+    }
 
     // Tell the manager to "commit" all the changes just made
     manager.Update();
@@ -601,21 +609,37 @@ void frmStatus::AddLogPane()
     {
         logFormatKnown = true;
         logHasTimestamp = true;
-        logList->AddColumn(_("Timestamp"), 100);
     }
     else if (connection->GetIsGreenplum())  
     {
         // Always %m|%u|%d|%p|%I|%X|:- (timestamp w/ millisec) for 3.2.x
-        // Always CSV formatted for 3.3
+        // Usually CSV formatted for 3.3
         logFormatKnown = true;
         logHasTimestamp = true;
-        logList->AddColumn(_("Timestamp"), 120);  // Room for millisecs
     }
 
-    if (logFormatKnown)
+
+    if (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8,2,13))
+    {
+        // Be ready for GPDB CSV format log file
+        logList->AddColumn(_("Timestamp"), 120);  // Room for millisecs
         logList->AddColumn(_("Level"), 35);
+        logList->AddColumn(_("Log entry"), 400);
+        logList->AddColumn(_("Connection"), 45);
+        logList->AddColumn(_("Cmd number"), 48);
+        logList->AddColumn(_("Dbname"), 48);
+        logList->AddColumn(_("Segment"), 45);
+    }
+    else    // Non-GPDB or non-CSV format log
+    {
+        if (logHasTimestamp)
+            logList->AddColumn(_("Timestamp"), 100);
 
-    logList->AddColumn(_("Log entry"), 800);
+        if (logFormatKnown)
+            logList->AddColumn(_("Level"), 35);
+
+        logList->AddColumn(_("Log entry"), 800);
+    }
 
     if (!connection->HasFeature(FEATURE_ROTATELOG))
         btnRotateLog->Disable();
@@ -938,7 +962,7 @@ void frmStatus::OnRefreshStatusTimer(wxTimerEvent &event)
     
     wxCriticalSectionLocker lock(gs_critsect);
 
-    connection->ExecuteVoid(wxT("SET log_statement='none';"));
+    connection->ExecuteVoid(wxT("SET log_statement='none';SET log_duration='off';"),false);
 
     long row=0;
     pgSet *dataSet1=connection->ExecuteSet(wxT("SELECT *,(SELECT min(pid) FROM pg_locks l1 WHERE GRANTED AND relation IN (SELECT relation FROM pg_locks l2 WHERE l2.pid=procpid AND NOT granted)) AS blockedby FROM pg_stat_activity ORDER BY procpid"));
@@ -963,7 +987,7 @@ void frmStatus::OnRefreshStatusTimer(wxTimerEvent &event)
                         break;
                 }
 
-                if (!itempid || itempid > pid)
+                if (!itempid || itempid > pid || row >= statusList->GetItemCount())
                 {
                     statusList->InsertItem(row, NumToStr(pid), 0);
                 }
@@ -1045,7 +1069,7 @@ void frmStatus::OnRefreshLocksTimer(wxTimerEvent &event)
     
     wxCriticalSectionLocker lock(gs_critsect);
 
-    connection->ExecuteVoid(wxT("SET log_statement='none';"));
+    connection->ExecuteVoid(wxT("SET log_statement='none';SET log_duration='off';"),false);
 
     long row=0;
     wxString sql;
@@ -1111,7 +1135,7 @@ void frmStatus::OnRefreshLocksTimer(wxTimerEvent &event)
                         break;
                 }
 
-                if (!itempid || itempid > pid)
+                if (!itempid || itempid > pid || lockList->GetItemCount() == 0)
                 {
                     lockList->InsertItem(row, NumToStr(pid), 0);
                 }
@@ -1192,7 +1216,7 @@ void frmStatus::OnRefreshXactTimer(wxTimerEvent &event)
     
     wxCriticalSectionLocker lock(gs_critsect);
 
-    connection->ExecuteVoid(wxT("SET log_statement='none';"));
+    connection->ExecuteVoid(wxT("SET log_statement='none';SET log_duration='off';"),false);
 
     long row=0;
     wxString sql = wxT("SELECT * FROM pg_prepared_xacts");
@@ -1276,7 +1300,18 @@ void frmStatus::OnRefreshLogTimer(wxTimerEvent &event)
     
     wxCriticalSectionLocker lock(gs_critsect);
 
-    connection->ExecuteVoid(wxT("SET log_statement='none';"));
+    connection->ExecuteVoid(wxT("SET log_statement='none';SET log_duration='off';"),false);
+
+    if (connection->GetLastResultError().sql_state == wxT("42501"))
+    {
+        // Don't have superuser privileges, so can't do anything with the log display
+        logTimer->Stop();
+        cbLogfiles->Disable();
+        btnRotateLog->Disable();
+        manager.GetPane(wxT("Logfile")).Show(false);
+        manager.Update();
+        return;
+    }
 
     long newlen=0;
 
@@ -1284,8 +1319,19 @@ void frmStatus::OnRefreshLogTimer(wxTimerEvent &event)
     {
         // freshly started
         logDirectory = connection->ExecuteScalar(wxT("SHOW log_directory"));
+        if (connection->GetLastResultError().sql_state == wxT("42501"))
+        {
+            // Don't have superuser privileges, so can't do anything with the log display
+            logTimer->Stop();
+            cbLogfiles->Disable();
+            btnRotateLog->Disable();
+            manager.GetPane(wxT("Logfile")).Show(false);
+            manager.Update();
+            return;
+        }
         if (fillLogfileCombo())
         {
+            savedPartialLine.Clear();
             cbLogfiles->SetSelection(0);
             wxCommandEvent ev;
             OnLoadLogfile(ev);
@@ -1430,9 +1476,26 @@ void frmStatus::addLogFile(const wxString &filename, const wxDateTime timestamp,
             skipFirst=false;
     }
 
+    // If GPDB 3.3 and later, log is normally in CSV format.  Let's get a whole log line before calling addLogLine,
+    // so we can do things smarter.
+
+    // PostgreSQL can log in CSV format, as well as regular format.  Normally, we'd only see 
+    // the regular format logs here, because pg_logdir_ls only returns those.  But if pg_logdir_ls is
+    // changed to return the csv format log files, we should handle it.
+
+    bool csv_log_format = filename.Right(4) == wxT(".csv");
+
+    if (csv_log_format && savedPartialLine.length() > 0)
+    {
+        if (read == 0)  // Starting at beginning of log file
+            savedPartialLine.clear();
+        else 
+            line = savedPartialLine;
+    }
 
     while (len > read)
     {
+        statusBar->SetStatusText(_("Reading log from server..."));
         pgSet *set=connection->ExecuteSet(wxT("SELECT pg_file_read(") + 
             connection->qtDbString(filename) + wxT(", ") + NumToStr(read) + wxT(", 50000)"));
         if (!set)
@@ -1447,6 +1510,7 @@ void frmStatus::addLogFile(const wxString &filename, const wxDateTime timestamp,
             delete set;
             break;
         }
+
         read += strlen(raw);
 
         wxString str;
@@ -1464,48 +1528,79 @@ void frmStatus::addLogFile(const wxString &filename, const wxDateTime timestamp,
             return;
         }
 
-        bool hasCr = (str.Right(1) == wxT("\n"));
+        if (csv_log_format)
+        {
+            // This will work for any DB using CSV format logs
 
-        wxStringTokenizer tk(str, wxT("\n"));
-            
-        bool gpdbformat = (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13));
+            if (logHasTimestamp)
+            {
+                // Right now, csv format logs from GPDB and PostgreSQL always start with a timestamp, so we count on that.
 
-        logList->Freeze();
-        if (gpdbformat)
-        {
-            // This would actually work for any DB assuming the log line prefix starts with %t or %m
-            wxString nextStr;
-            while (tk.HasMoreTokens())
+                // And the only reason we need to do that is to make sure we are in sync.  
+
+                // Bad things happen if we start in the middle of a 
+                // double-quoted string, as we would never find a correct line terminator!
+
+                // In CSV logs, the first field must be a Timestamp, so must start with "2009" or "201" or "202" (at least for the next 20 years).
+                if (str.length() > 4 && str.Left(4) != wxT("2009") && str.Left(3) != wxT("201") && str.Left(3) != wxT("202"))
+                {
+                    wxLogNotice(wxT("Log line does not start with timestamp: %s \n"), str.Mid(0,100).c_str());
+                    // Something isn't right, as we are not at the beginning of a csv log record.
+                    // We should never get here, but if we do, try to handle it in a smart way.
+                    str = str.Mid(str.Find(wxT("\n20"))+1);  // Try to re-sync.
+                }
+            }
+
+            CSVLineTokenizer tk(str);
+
+            logList->Freeze();
+
+            while (tk.HasMoreLines())
             {
-                str = nextStr;
-                nextStr = tk.GetNextToken();
-                if (str.Length() == 0)
-                    continue;
+                line.Clear();
 
-                // The first field must be a Timestamp, so must start with "2009" or "201" (at least for the next 10 years).
-                if (skipFirst && (str.Left(4) != wxT("2009") && str.Left(3) != wxT("201")))
+                bool partial;
+                str = tk.GetNextLine(partial);
+                if (partial)
                 {
-                    // Something isn't right, as we are not at the beginning of a log record.
-                    skipFirst = false;
-                    continue;
+                    line = str; // Start of a log line, but not complete.  Loop back, Read more data.
+                    break;
                 }
 
-                // The first field must be a Timestamp, so must start with "2009" or "201" (at least for the next 10 years).
-                // If nextStr doesn't start with that, it must be a continuation of this line.
-                if (nextStr.Left(4) != wxT("2009") && nextStr.Left(3) != wxT("201"))
+                // Some extra debug checking, assuming csv logs line start with timestamps.
+                // Not really necessary, but it is good for debugging if something isn't right.
+                if (logHasTimestamp)
                 {
-                    nextStr = str + wxT("\n") + nextStr;
-                    continue;
+                    // The first field must be a Timestamp, so must start with "2009" or "201" or "202" (at least for the next 20 years).
+                    // This is just an extra check to make sure we haven't gotten out of sync with the log.
+                    if (str.length() > 5 && str.Left(4) != wxT("2009") && str.Left(3) != wxT("201") && str.Left(3) != wxT("202"))
+                    {
+                        // BUG:  We are out of sync on the log
+                        wxLogNotice(wxT("Log line does not start with timestamp: %s\n"), str.c_str());;
+                    }
+                    else if (str.length() < 20)
+                    {
+                        // BUG:  We are out of sync on the log, or the log is garbled
+                        wxLogNotice(wxT("Log line too short: %s\n"), str.c_str());
+                    }
                 }
 
-                if (tk.HasMoreTokens() || hasCr)
-                    addLogLine(str.Trim());
-                else
-                    line = str;
+                // Looks like we have a good complete CSV log record.
+                addLogLine(str.Trim(), true, true);
             }
+
+            logList->Thaw();
         }
         else
         {
+            // Non-csv format log file
+
+            bool hasCr = (str.Right(1) == wxT("\n"));
+
+            wxStringTokenizer tk(str, wxT("\n"));
+
+            logList->Freeze();
+
             while (tk.HasMoreTokens())
             {
                 str = tk.GetNextToken();
@@ -1521,225 +1616,315 @@ void frmStatus::addLogFile(const wxString &filename, const wxDateTime timestamp,
                 else
                     line = str;
             }
+
+            logList->Thaw();
         }
-        logList->Thaw();
     }
+    
+    savedPartialLine.clear();
+
     if (!line.IsEmpty())
-        addLogLine(line.Trim());
+    {
+        // We finished reading to the end of the log file, but still have some data left
+        if (csv_log_format)
+        {
+            savedPartialLine = line;    // Save partial log line for next read of the data file.
+            line.Clear();
+        }
+        else
+            addLogLine(line.Trim());
+    }
+        
 }
 
 
-void frmStatus::addLogLine(const wxString &str, bool formatted)
+void frmStatus::addLogLine(const wxString &str, bool formatted, bool csv_log_format)
 {
     int row=logList->GetItemCount();
+
     if (!logFormatKnown)
         logList->AppendItem(-1, str);
-    else
+    else if ((!csv_log_format) && str.Find(':') < 0)
     {
-        if (connection->GetIsGreenplum() && connection->BackendMinimumVersion(8, 2, 13))
+        // Must be a continuation of a previous line.
+        logList->InsertItem(row, wxEmptyString, -1);
+        logList->SetItem(row, 2, str);
+    }
+    else if (!formatted)
+    {
+        // Not from a log, from pgAdmin itself.
+        logList->InsertItem(row, wxEmptyString, -1);
+        logList->SetItem(row, 1, str.BeforeFirst(':'));
+        logList->SetItem(row, 2, str.AfterFirst(':'));
+    }
+    else // formatted log
+    { 
+        if (csv_log_format)
         {
-            // Greenplum 3.3 release and later:  log is in CSV format
+            // Log is in CSV format (GPDB 3.3 and later, or Postgres if only csv log enabled)
+            // In this case, we are always supposed to have a complete log line in csv format in str when called.
 
-            if (!formatted)
+            if (logHasTimestamp && (str.Length() < 20 || (logHasTimestamp && (str[0] != wxT('2') || str[1] != wxT('0')))))
             {
-                // Not from log, from pgAdmin itself.
+                // Log line too short or does not start with an expected timestamp... 
+                // Must be a continuation of the previous line or garbage,
+                // or we are out of sync in our CSV handling.
+                // We shouldn't ever get here.
                 logList->InsertItem(row, wxEmptyString, -1);
-                logList->SetItem(row, 1, str.BeforeFirst(':'));
-                logList->SetItem(row, 2, str.AfterFirst(':').Mid(2));
+                logList->SetItem(row, 2, str);
             }
             else
             {
-                wxStringTokenizer tk(str, wxT(","));
-                wxString logTime = tk.GetNextToken();
-                if (logTime.Length() < 20 || logTime[0] != wxT('2') || logTime[1] != wxT('0'))
-                {
-                    // Log line does not start with a timestamp... Must be a continuation of the previous line or garbage.
-                    logList->InsertItem(row, wxEmptyString, -1);
-                    logList->SetItem(row, 2, str);
-                }
-                else
+                CSVTokenizer tk(str);
+
+                bool gpdb = connection->GetIsGreenplum();
+
+                // Get the fields from the CSV log. 
+                wxString logTime = tk.GetNextToken();         
+                wxString logUser = tk.GetNextToken();
+                wxString logDatabase = tk.GetNextToken();
+                wxString logPid = tk.GetNextToken();
+
+                wxString logSession;
+                wxString logCmdcount;
+                wxString logSegment;
+
+                if (gpdb)
                 {
-                    wxString logUser = tk.GetNextToken();
-                    wxString logDatabase = tk.GetNextToken();
-                    wxString logPid = tk.GetNextToken();
-                    wxString logThread =  tk.GetNextToken();
+                    wxString logThread =  tk.GetNextToken();        // GPDB specific
                     wxString logHost = tk.GetNextToken();
-                    wxString logPort = tk.GetNextToken();
+                    wxString logPort = tk.GetNextToken();           // GPDB (Postgres puts port with Host)
                     wxString logSessiontime = tk.GetNextToken();
                     wxString logTransaction = tk.GetNextToken();
-                    wxString logSession = tk.GetNextToken();
-                    wxString logCmdcount = tk.GetNextToken();
-                    wxString logSegment = tk.GetNextToken();
+                    logSession = tk.GetNextToken();
+                    logCmdcount = tk.GetNextToken();
+                    logSegment = tk.GetNextToken();
                     wxString logSlice = tk.GetNextToken();
                     wxString logDistxact = tk.GetNextToken();
                     wxString logLocalxact = tk.GetNextToken();
                     wxString logSubxact = tk.GetNextToken();
-                    wxString logSeverity = tk.GetNextToken();
-                    wxString logState = tk.GetNextToken();
-                    wxString logMessage = tk.GetNextToken();
-                    while (logMessage.Length() > 2 && logMessage[logMessage.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logMessage = logMessage + wxT(",") + tk.GetNextToken();
-                    wxString logDetail = tk.GetNextToken();
-                    while (logDetail.Length() > 2 && logDetail[logDetail.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logDetail = logDetail + wxT(",") + tk.GetNextToken();
-                    wxString logHint = tk.GetNextToken();
-                    while (logHint.Length() > 2 && logHint[logHint.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logHint = logHint + wxT(",") + tk.GetNextToken();
-                    wxString logQuery = tk.GetNextToken();
-                    while (logQuery.Length() > 2 && logQuery[logQuery.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logQuery = logQuery + wxT(",") + tk.GetNextToken();
-                    wxString logQuerypos = tk.GetNextToken();
-                    wxString logContext = tk.GetNextToken();
-                    wxString logDebug = tk.GetNextToken();
-                    while (logDebug.Length() > 2 && logDebug[logDebug.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logDebug = logDebug + wxT(",") + tk.GetNextToken();
-                    wxString logCursorpos = tk.GetNextToken();
-                    wxString logFunction = tk.GetNextToken();
+                }
+                else
+                {
+                    wxString logHost = tk.GetNextToken();       // Postgres puts port with Hostname
+                    logSession = tk.GetNextToken();
+                    wxString logLineNumber = tk.GetNextToken();
+                    wxString logPsDisplay = tk.GetNextToken();
+                    wxString logSessiontime = tk.GetNextToken();
+                    wxString logVXid = tk.GetNextToken();
+                    wxString logTransaction = tk.GetNextToken();
+                }
+                
+                wxString logSeverity = tk.GetNextToken();
+                wxString logState = tk.GetNextToken();
+                wxString logMessage = tk.GetNextToken();
+                wxString logDetail = tk.GetNextToken();
+                wxString logHint = tk.GetNextToken();
+                wxString logQuery = tk.GetNextToken();
+                wxString logQuerypos = tk.GetNextToken();
+                wxString logContext = tk.GetNextToken();
+                wxString logDebug = tk.GetNextToken();
+                wxString logCursorpos = tk.GetNextToken();
+
+                wxString logStack;
+                if (gpdb)
+                {
+                    wxString logFunction = tk.GetNextToken();       // GPDB.  Postgres puts func, file, and line together
                     wxString logFile = tk.GetNextToken();
                     wxString logLine = tk.GetNextToken();
-                    wxString logStack = tk.GetNextToken();
-                    while (logStack.Length() > 2 && logStack[logStack.Length()-1] != wxT('\"') && tk.HasMoreTokens())
-                        logStack = logStack + wxT(",") + tk.GetNextToken();
+                    logStack = tk.GetNextToken();                   // GPDB only.
+                }
+                else
+                    wxString logFuncFileLine = tk.GetNextToken();  
 
-                    logSeverity = logSeverity.AfterFirst('\"').BeforeLast('\"');
-                    logMessage = logMessage.AfterFirst('\"').BeforeLast('\"');
-                    logStack = logStack.AfterFirst('\"').BeforeLast('\"');
+                logList->InsertItem(row, logTime, -1);      // Insert timestamp (with time zone)
+            
+                logList->SetItem(row, 1, logSeverity);
 
-                    wxStringTokenizer lm(logMessage,wxT("\n"));
-                    wxStringTokenizer ls(logStack,wxT("\n"));
-                    
-                    logList->InsertItem(row, logTime, -1);   // Insert timestamp
-                    
-                    
-                    logList->SetItem(row, 1, logSeverity);
-                    logList->SetItem(row, 2, lm.GetNextToken());
-                    
+                // Display the logMessage, breaking it into lines
+                wxStringTokenizer lm(logMessage,wxT("\n"));
+                logList->SetItem(row, 2, lm.GetNextToken());
 
-                    while (lm.HasMoreTokens())
+                logList->SetItem(row, 3, logSession);
+                logList->SetItem(row, 4, logCmdcount);
+                logList->SetItem(row, 5, logDatabase);
+                if ((!gpdb) || (logSegment.length() > 0 && logSegment != wxT("seg-1")))
+                {
+                    logList->SetItem(row, 6, logSegment);
+                }
+                else
+                {
+                    // If we are reading the masterDB log only, the logSegment won't
+                    // have anything useful in it.  Look in the logMessage, and see if the
+                    // segment info exists in there.  It will always be at the end.
+                    if (logMessage.length() > 0 && logMessage[logMessage.length()-1] == wxT(')'))
                     {
-                        int controw=logList->GetItemCount();
-                        logList->InsertItem(controw, wxEmptyString, -1);
-                        logList->SetItem(controw, 2, lm.GetNextToken());
+                        int segpos = -1;
+                        segpos = logMessage.Find(wxT("(seg"));
+                        if (segpos <= 0)
+                            segpos = logMessage.Find(wxT("(mir"));
+                        if (segpos > 0)
+                        {
+                            logSegment = logMessage.Mid(segpos+1);
+                            if (logSegment.Find(wxT(' ')) > 0)
+                                logSegment = logSegment.Mid(0,logSegment.Find(wxT(' ')));
+                            logList->SetItem(row, 6, logSegment);
+                        }
                     }
-                    while (ls.HasMoreTokens())
+                }
+                
+                // The rest of the lines from the logMessage
+                while (lm.HasMoreTokens())
+                {
+                    int controw=logList->GetItemCount();
+                    logList->InsertItem(controw, wxEmptyString, -1);
+                    logList->SetItem(controw, 2, lm.GetNextToken());
+                }
+
+                // Add the detail
+                wxStringTokenizer ld(logDetail,wxT("\n"));
+                while (ld.HasMoreTokens())
+                {
+                    int controw=logList->GetItemCount();
+                    logList->InsertItem(controw, wxEmptyString, -1);
+                    logList->SetItem(controw, 2, ld.GetNextToken());
+                }
+
+                // And the hint
+                wxStringTokenizer lh(logHint,wxT("\n"));
+                while (lh.HasMoreTokens())
+                {
+                    int controw=logList->GetItemCount();
+                    logList->InsertItem(controw, wxEmptyString, -1);
+                    logList->SetItem(controw, 2, lh.GetNextToken());
+                }
+
+                if (logDebug.length() > 0)
+                {
+                    wxString logState3 = logState.Mid(0,3);
+                    if (logState3 == wxT("426") || logState3 == wxT("22P") || logState3 == wxT("427") 
+                        || logState3 == wxT("42P") || logState3 == wxT("458") 
+                        || logMessage.Mid(0,9) == wxT("duration:") || logSeverity == wxT("FATAL") || logSeverity == wxT("PANIC"))
                     {
-                        int controw=logList->GetItemCount();
-                        logList->InsertItem(controw, wxEmptyString, -1);
-                        logList->SetItem(controw, 2, ls.GetNextToken());
+                        // If not redundant, add the statement from the debug_string
+                        wxStringTokenizer lh(logDebug,wxT("\n"));
+                        if (lh.HasMoreTokens())
+                        {
+                            int controw=logList->GetItemCount();
+                            logList->InsertItem(controw, wxEmptyString, -1);
+                            logList->SetItem(controw, 2, wxT("statement: ") + lh.GetNextToken());
+                        }
+                        while (lh.HasMoreTokens())
+                        {
+                            int controw=logList->GetItemCount();
+                            logList->InsertItem(controw, wxEmptyString, -1);
+                            logList->SetItem(controw, 2, lh.GetNextToken());
+                        }
                     }
-                   
                 }
+
+                if (gpdb)
+                    if (logSeverity == wxT("PANIC") || 
+                        (logSeverity == wxT("FATAL") && logState != wxT("57P03") && logState != wxT("53300")))
+                    {
+                        // If this is a severe error, add the stack trace.
+                        wxStringTokenizer ls(logStack,wxT("\n"));
+                        if (ls.HasMoreTokens())
+                        {
+                            int controw=logList->GetItemCount();
+                            logList->InsertItem(controw, wxEmptyString, -1);
+                            logList->SetItem(controw, 1, wxT("STACK"));
+                            logList->SetItem(controw, 2, ls.GetNextToken());
+                        }
+                        while (ls.HasMoreTokens())
+                        {
+                            int controw=logList->GetItemCount();
+                            logList->InsertItem(controw, wxEmptyString, -1);
+                            logList->SetItem(controw, 2, ls.GetNextToken());
+                        }
+                    }
             }
         }
         else if (connection->GetIsGreenplum())
         {
-            // Greenplum 3.2 and before.
+            // Greenplum 3.2 and before.  log_line_prefix =  "%m|%u|%d|%p|%I|%X|:-"
+
+            wxString logSeverity;
+            // Skip prefix, get message.  In GPDB, always follows ":-".
+            wxString rest = str.Mid(str.Find(wxT(":-"))+1) ;
+            if (rest.Length() > 0 && rest[0] == wxT('-'))
+                rest = rest.Mid(1);
+
+            // Separate loglevel from message
+
+            if (rest.Length() > 1 && rest[0] != wxT(' ') && rest.Find(':') > 0)
+            {
+                logSeverity = rest.BeforeFirst(':');
+                rest = rest.AfterFirst(':').Mid(2);
+            }
 
-            if (str.Find(':') < 0)
+            wxString ts = str.BeforeFirst(logFormat.c_str()[logFmtPos+2]);
+            if (ts.Length() < 20  || (logHasTimestamp && (ts.Left(2) != wxT("20") || str.Find(':') < 0)))
             {
-                // Must be a continuation of a previous line, or a message from pgAdmin itself.
-                // Not sure if we ever get here, since lines from the log always have a timestamp at the start.
+                // No Timestamp?  Must be a continuation of a previous line?  
+                // Not sure if it is possible to get here.
                 logList->InsertItem(row, wxEmptyString, -1);
-                logList->SetItem(row, 2, str);
+                logList->SetItem(row, 2, rest);
+            }
+            else if (logSeverity.Length() > 1)
+            {
+                // Normal case:  Start of a new log record.
+                logList->InsertItem(row, ts, -1);
+                logList->SetItem(row, 1, logSeverity);
+                logList->SetItem(row, 2, rest);   
             }
             else
             {
-                wxString logSeverity;
-                // Skip prefix, get message
-                wxString rest = str.Mid(str.Find(wxT(":-"))+1) ;
-                if (rest[0] == wxT('-'))
-                    rest = rest.Mid(1);
-
-                // Separate loglevel from message
+                // Continuation of previous line
+                logList->InsertItem(row, wxEmptyString, -1);
+                logList->SetItem(row, 2, rest);
+            }
+        }
+        else
+        {
+            // All Non-csv-format non-GPDB PostgreSQL systems.
 
+            wxString rest;
 
-                if (rest[0] != wxT(' ') && rest.Find(':') > 0)
-                {
-                    logSeverity = rest.BeforeFirst(':');
-                    rest = rest.AfterFirst(':').Mid(2);
-                }
-
+            if (logHasTimestamp)
+            {
                 if (formatted)
                 {
-                    wxString ts = str.BeforeFirst(logFormat.c_str()[logFmtPos+2]);
-                    if (ts.Length() < 20  || ts.Left(2) != wxT("20") || str.Find(':') < 0)
-                    {
-                        // No Timestamp?  Must be a continuation of a previous line?  
-                        // Not sure if it is possible to get here.
-                        logList->InsertItem(row, wxEmptyString, -1);
-                        logList->SetItem(row, 2, rest);
-                    }
-                    else if (logSeverity.Length() > 1)
-                    {
-                        // Normal case:  Start of a new log record.
-                        logList->InsertItem(row, ts, -1);
-                        logList->SetItem(row, 1, logSeverity);
-                        logList->SetItem(row, 2, rest);   
-                    }
-                    else
-                    {
-                        // Continuation of previous line
-                        logList->InsertItem(row, wxEmptyString, -1);
-                        logList->SetItem(row, 2, rest);
-                    }
+                    rest = str.Mid(logFmtPos + 22).AfterFirst(':');
+                    wxString ts=str.Mid(logFmtPos, str.Length()-rest.Length() - logFmtPos -1);
+
+                    int pos = ts.Find(logFormat.c_str()[logFmtPos+2], true);
+                    logList->InsertItem(row, ts.Left(pos), -1);
+                    logList->SetItem(row, 1, ts.Mid(pos + logFormat.Length() - logFmtPos -2));
+                    logList->SetItem(row, 2, rest.Mid(2));
                 }
-                else // Not from log, from pgAdmin itself?
+                else
                 {
                     logList->InsertItem(row, wxEmptyString, -1);
                     logList->SetItem(row, 1, str.BeforeFirst(':'));
                     logList->SetItem(row, 2, str.AfterFirst(':').Mid(2));
                 }
             }
-            
-        }
-        else
-        {
-            // All Non-GPDB PostgreSQL systems.
-
-            if (str.Find(':') < 0)
-            {
-                logList->InsertItem(row, wxEmptyString, -1);
-                logList->SetItem(row, (logHasTimestamp ? 2 : 1), str);
-            }
             else
             {
-                wxString rest;
+                if (formatted)
+                    rest = str.Mid(logFormat.Length());
+                else
+                    rest = str;
 
-                if (logHasTimestamp)
-                {
-                    if (formatted)
-                    {
-                        rest = str.Mid(logFmtPos + 22).AfterFirst(':');
-                        wxString ts=str.Mid(logFmtPos, str.Length()-rest.Length() - logFmtPos -1);
+                int pos = rest.Find(':');
 
-                        int pos = ts.Find(logFormat.c_str()[logFmtPos+2], true);
-                        logList->InsertItem(row, ts.Left(pos), -1);
-                        logList->SetItem(row, 1, ts.Mid(pos + logFormat.Length() - logFmtPos -2));
-                        logList->SetItem(row, 2, rest.Mid(2));
-                    }
-                    else
-                    {
-                        logList->InsertItem(row, wxEmptyString, -1);
-                        logList->SetItem(row, 1, str.BeforeFirst(':'));
-                        logList->SetItem(row, 2, str.AfterFirst(':').Mid(2));
-                    }
-                }
+                if (pos < 0)
+                    logList->InsertItem(row, rest, -1);
                 else
                 {
-                    if (formatted)
-                        rest = str.Mid(logFormat.Length());
-                    else
-                        rest = str;
-
-                    int pos = rest.Find(':');
-
-                    if (pos < 0)
-                        logList->InsertItem(row, rest, -1);
-                    else
-                    {
-                        logList->InsertItem(row, rest.BeforeFirst(':'), -1);
-                        logList->SetItem(row, 1, rest.AfterFirst(':').Mid(2));
-                    }
+                    logList->InsertItem(row, rest.BeforeFirst(':'), -1);
+                    logList->SetItem(row, 1, rest.AfterFirst(':').Mid(2));
                 }
             }
         }
index e9843bd0e4ca44f3cd51a3831127031f7881f413..e59dea2047ec09c4ecc63a4e473fbae839b1951b 100644 (file)
@@ -101,6 +101,9 @@ private:
 
     wxDateTime logfileTimestamp, latestTimestamp;
     wxString logDirectory, logfileName;
+
+    wxString savedPartialLine;
+
     bool showCurrent, isCurrent;
     
     long backend_pid;
@@ -175,7 +178,7 @@ private:
 
     void addLogFile(wxDateTime *dt, bool skipFirst);
     void addLogFile(const wxString &filename, const wxDateTime timestamp, long len, long &read, bool skipFirst);
-    void addLogLine(const wxString &str, bool formatted=true);
+    void addLogLine(const wxString &str, bool formatted=true, bool csv_log_format=false);
 
        void checkConnection();
     
index 6221faa452bb314884ee1ee71efda71ae6ddd765..f7152d512bc77b7ba0649b36aa82b9ec6f4a2c06 100644 (file)
 #include "slony/slSubscription.h"
 #include "slony/slTable.h"
 
+#include "utils/csvfiles.h"
 #include "utils/factory.h"
 #include "utils/favourites.h"
 #include "utils/macros.h"
diff --git a/pgadmin/include/utils/csvfiles.h b/pgadmin/include/utils/csvfiles.h
new file mode 100644 (file)
index 0000000..7bde2c8
--- /dev/null
@@ -0,0 +1,53 @@
+//////////////////////////////////////////////////////////////////////////
+//
+// pgAdmin III - PostgreSQL Tools
+// RCS-ID:      $Id:  $
+// Copyright (C) 2002 - 2009, The pgAdmin Development Team
+// This software is released under the BSD Licence
+//
+// csvfiles.h - CSV file parsing
+//
+//////////////////////////////////////////////////////////////////////////
+
+#ifndef CSVFILES_H
+#define CSVFILES_H
+
+// PostgreSQL and GPDB now support CSV format logs.
+// So, we need a way to parse the CSV files into lines, and lines into tokens (fields).
+
+#include <wx/wx.h>
+
+class CSVTokenizer : public wxObject
+{
+public:
+    CSVTokenizer(const wxString& str): m_string(str), m_pos(0) { }
+
+    bool HasMoreTokens() const;
+
+    // Get the next token (CSV field). Will return empty string if !HasMoreTokens()
+    wxString GetNextToken();
+
+protected:
+
+    const wxString m_string;        // the string we tokenize into fields
+    size_t   m_pos;                 // the current position in m_string
+};
+
+class CSVLineTokenizer : public wxObject
+{
+public:
+    CSVLineTokenizer(const wxString& str): m_string(str), m_pos(0) { }
+
+    bool HasMoreLines() const;
+
+    // Get the next line.  Will return empty string if !HasMoreLines().
+    // partial is set "true" if the last line returned was not a complete
+    // line (no newline char at end).
+    wxString GetNextLine(bool & partial);
+
+protected:
+
+    const wxString m_string;        // the string we tokenize into lines
+    size_t   m_pos;                 // the current position in m_string
+};
+#endif
index 84c8b28fc4276e4dc95ebefdb1517126ddcf6212..a811bc283c5a9fad9f5e72e151336aafff75a366 100644 (file)
@@ -10,6 +10,7 @@
 #######################################################################
 
 pgadmin3_SOURCES += \
+       $(srcdir)/include/utils/csvfiles.h \
        $(srcdir)/include/utils/factory.h \
        $(srcdir)/include/utils/favourites.h \
        $(srcdir)/include/utils/md5.h \
index 25c9d5033ae216eb1b6f708cb032c1bec523c6d6..b95e1e6a4ff6320b2ef29a3bbccc6d5f256f4336 100644 (file)
                        <Filter\r
                                Name="utils"\r
                                >\r
+                               <File\r
+                                       RelativePath=".\include\utils\csvfiles.h"\r
+                                       >\r
+                               </File>\r
                                <File\r
                                        RelativePath=".\include\utils\factory.h"\r
                                        >\r
                <Filter\r
                        Name="utils"\r
                        >\r
+                       <File\r
+                               RelativePath=".\utils\csvfiles.cpp"\r
+                               >\r
+                       </File>\r
                        <File\r
                                RelativePath=".\utils\factory.cpp"\r
                                >\r
diff --git a/pgadmin/utils/csvfiles.cpp b/pgadmin/utils/csvfiles.cpp
new file mode 100644 (file)
index 0000000..19b0709
--- /dev/null
@@ -0,0 +1,140 @@
+//////////////////////////////////////////////////////////////////////////
+//
+// pgAdmin III - PostgreSQL Tools
+// RCS-ID:      $Id:  $
+// Copyright (C) 2002 - 2009, The pgAdmin Development Team
+// This software is released under the BSD Licence
+//
+// csvfiles.cpp - CSV file parsing
+//
+//////////////////////////////////////////////////////////////////////////
+
+#include "pgAdmin3.h"
+#include "utils/sysLogger.h"
+#include "utils/csvfiles.h"
+
+// PostgreSQL and GPDB now support CSV format logs.
+// So, we need a way to parse the CSV files into lines, and lines into tokens (fields).
+
+bool CSVTokenizer::HasMoreTokens() const
+{
+    if ( m_string.length() > 0)
+    {
+        if ( m_pos >= m_string.length())
+            return false;
+
+        if ( m_string.find_first_not_of(wxT(','), m_pos) != wxString::npos )
+            // there are non delimiter characters left, so we do have more tokens
+            return true;
+
+        if (m_string[m_pos] == wxT('\n'))
+            return false;
+    }
+    return m_pos == 0 && !m_string.empty();
+}
+
+wxString CSVTokenizer::GetNextToken()
+{
+    wxString token;
+
+    if ( !HasMoreTokens() )
+        return token;
+
+    // skip leading blanks if not quoted.
+    while (m_pos < m_string.length() && m_string[m_pos] == wxT(' '))
+        m_pos ++;
+
+    // Are we a quoted field?  Must handle this special.
+    bool quoted_string = (m_string[m_pos] == wxT('\"'));
+    bool inquote = false;
+
+    size_t pos = m_pos;
+
+    // find the end of this token.
+    for (; pos < m_string.length(); pos++)
+    {
+        if (quoted_string && m_string[pos] == wxT('\"'))
+            inquote = !inquote;
+
+        if (!inquote)
+        {
+            // Check to see if we have found the end of this token.
+            // Tokens normally end with a ',' delimiter.
+            if (m_string[pos] == wxT(','))
+                break;
+
+            // Last token is delimited by '\n' or by end of string.
+            if (m_string[pos] == wxT('\n') && pos == m_string.length()-1)
+                break;
+        }
+    }
+
+    if (quoted_string && !inquote)
+    {
+        token.assign(m_string, m_pos + 1, pos - m_pos - 2);  // Remove leading and trailing quotes
+
+        // Remove double doublequote chars, replace with single doublequote chars
+        token.Replace(wxT("\"\""),wxT("\""),true);
+    }
+    else
+        token.assign(m_string, m_pos, pos - m_pos);
+
+    if (quoted_string && inquote)
+            wxLogNotice(wxT("unterminated double quoted string: %s\n"),token);
+
+    m_pos = pos + 1;    // Skip token and delimiter
+
+    if (m_pos > m_string.length())  // Perhaps no delimiter if at end of string if orig string didn't have '\n'.
+        m_pos = m_string.length();
+
+    return token;
+}
+
+bool CSVLineTokenizer::HasMoreLines() const
+{
+    if ( m_string.find_first_not_of(wxT('\n'), m_pos) != wxString::npos )
+        // there are non line-end characters left, so we do have more lines
+        return true;
+    return false;
+}
+
+wxString CSVLineTokenizer::GetNextLine(bool & partial)
+{
+    wxString token;
+    partial = true;
+
+    if ( !HasMoreLines() )
+        return token;
+
+    // find the end of this line.  CSV lines end in "\n", but 
+    // CSV lines may have "\n" chars inside double-quoted strings, so we need to find that out.
+
+    bool inquote = false;
+    for (size_t pos = m_pos; pos < m_string.length(); pos++)
+    {
+        if (m_string[pos] == wxT('\"'))
+            inquote = !inquote;
+
+        if (m_string[pos] == wxT('\n') && !inquote)
+        {
+            // Good, we found a complete log line terminated 
+            // by "\n", and the "\n" wasn't in a quoted string.
+
+            size_t len = pos - m_pos + 1;   // return the line, including the trailing "\n"
+            token.assign(m_string, m_pos, len);
+            m_pos = pos + 1;                // point to next line.
+            partial = false;
+            return token;
+        }
+    }
+
+    // no more delimiters, so the line is everything till the end of
+    // string, but we don't have all of the CSV the line... Some must still be coming.
+     
+    token.assign(m_string, m_pos, wxString::npos);
+    partial = true;
+
+    m_pos = m_string.length();
+  
+    return token;
+}
index f615a199fb145da8305865a37627916609314e91..06af8c63da8241740ec3ab106ab372222221ea9f 100644 (file)
@@ -10,6 +10,7 @@
 #######################################################################
 
 pgadmin3_SOURCES += \
+       $(srcdir)/utils/csvfiles.cpp \
        $(srcdir)/utils/factory.cpp \
        $(srcdir)/utils/favourites.cpp \
        $(srcdir)/utils/md5.cpp \