Dataguard alike using bash and cron

Oracle EE is quite more expensive than SE, however sometimes you need a standby replica “just in case”.

This is how you get a standby replica on a different server, without breaking any Oracle licensing rules:

1) the following script has to be on both servers. It calls itself on remote.
2) set up a cronjob to launch the script on the primary server.
3) Check the log.

I must say that this script needs some big cleaning and ordering, but it works.

#!/bin/bash +x
#joanmarcrieraATgmailDOCcom

#TODO:: fix the output processing when DEBUG=1
#TODO:: too many paths that must become variables


STB_SERVER=oranode2                                     #the Standby database server # needs passwordless authentication to user oracle
LOG_FILE=/home/oracle/scripts/db_status.log             #

#load the environment variables
. ~/.bash_profile

# When debug=1 the scripts must be executed by hand , and most functions will not work as expected. Processing the functions output must be fixed
DEBUG=0

# is the local database OK ?
function check_status ()
{
[[ $DEBUG -eq 1 ]] && echo "DEBUG=check_Status"
#expected status "ok=1 ko=0"
INSTANCE_STATUS=$(echo "select STATUS from v\$instance; " | sqlplus -S / as sysdba|grep -E "MOUNTED|OPEN"|wc -l )
INSTANCE_ARCHIVER=$(echo "select ARCHIVER from v\$instance; " | sqlplus -S / as sysdba|grep "STARTED"| wc -l)
INSTANCE_DB_STATUS=$(echo "select DATABASE_STATUS from v\$instance; " | sqlplus -S / as sysdba|grep "ACTIVE" |wc -l)
INSTANCE_ACTIVE_STATE=$(echo "select ACTIVE_STATE from v\$instance; " | sqlplus -S / as sysdba|grep "NORMAL"|wc -l)

[[ $INSTANCE_STATUS -eq 1 ]] && [[ $INSTANCE_ARCHIVER -eq 1 ]] && [[ $INSTANCE_DB_STATUS -eq 1 ]] && [[ $INSTANCE_ACTIVE_STATE -eq 1 ]] && echo "STATUS = OK" || echo "STATUS = KO"

}

#check tns ping .
function tnsping_status ()
{
[[ $DEBUG -eq 1 ]] && echo "DEBUG=tnsping_status"
TNSPING=$(tnsping eypesa|tail -n1 |cut -d' ' -f1 |grep "OK"|wc -l)
[[ $TNSPING -eq 1 ]] && echo "STATUS = OK" || echo "STATUS = KO"
}

# get value for logcount on local database (open mode)
function archive_log_count ()
{
[[ $DEBUG -eq 1 ]] && echo "DEBUG=archive_log_count"
#don't return current, because is not finished. so return the last already switched.
#select max(SEQUENCE#) from V$ARCHIVED_LOG;
ARCHIVE_LOG=$(echo "select max(SEQUENCE#) from V\$ARCHIVED_LOG;" |sqlplus -S / as sysdba|tail -n2|head -n1|tr -d ' \t\r\f')
echo "LAST = $ARCHIVE_LOG"
}

#ask database to check for its local logcount  (standby mode)
function archive_log_count_stb ()
{
#select max(recid) from v$log_history;"
ARCHIVE_LOG_STB=$(echo "select max(recid) from v\$log_history;"|sqlplus -S / as sysdba |tail -n2 |head -n1|tr -d ' \t\r\f')
echo "LAST_STB = $ARCHIVE_LOG_STB"
}

#ask for remote logcount (standby mode)
function archive_log_count_stb_remote ()
{
ssh $STB_SERVER ~/scripts/db_status.sh archive_log_stb
}

#send new archive files to STB_SERVER
function send_files ()
{
#seq 452 $(expr 454 - 1)
FROM_FILE=$(ssh $STB_SERVER ~/scripts/db_status.sh archive_log_stb |cut -d'=' -f2)
TO_FILE=$(~/scripts/db_status.sh archive_log |cut -d'=' -f2)

[[ $FROM_FILE -eq $TO_FILE ]] && echo "SAME STATUS ON BOTH SIDES. " && exit 0

#files to send FROM TO -1
for FILE in $(seq $FROM_FILE $TO_FILE)
do
rsync -v /u01/eypesa/ORADATA/DB_ARCHIVE/1_${FILE}_* $STB_SERVER:/u01/eypesa/ORADATA/DB_ARCHIVE
done
}

#set remote database to recover automatically with archive logs already sent
function recover_standby_remote ()
{
#start recovery on standby
ssh $STB_SERVER ~/scripts/db_status.sh recover_local_standby
}

#recover with local archive logs
function recover_standby_local ()
{
sqlplus -S / as sysdba @/home/oracle/scripts/recover_standby_database.sql
}

#switch logfile  (debug purposes)
function switch_local ()
{
echo "alter system switch logfile;" |sqlplus -S / as sysdba
}

#delete old archive log files, this is used on STB_DATABASE because it does not have rman to clean it.
function clean_old_files_remote ()
{
ssh $STB_SERVER "find /u01/eypesa/ORADATA/DB_ARCHIVE -mtime +7 -exec rm -rf {} \;"
}

#procedure when launched by cron.
function cron_mode ()
{
exec 1>> $LOG_FILE
exec 2>> $LOG_FILE
echo "CRON EXECUTION : $(date +%Y%m%d%H%M)"
#check status
check_status
tnsping_status
archive_log_count
archive_log_count_stb_remote
#send files
send_files
#recover
recover_standby_remote
clean_old_files_remote

echo "###### END CRON EXECUTION : $(date +%Y%m%d%H%M)"

}

case $1 in
        status)
                check_status
                ;;
        tnsping)
                tnsping_status
                ;;
        archive_log)
                archive_log_count
                ;;
        archive_log_stb)
                archive_log_count_stb
                ;;
        send_files)
                send_files
                ;;
        recover_remote_standby)
                recover_standby_remote
                ;;
        recover_local_standby)
                recover_standby_local
                ;;
        switch_logfile_local)
                switch_local
                ;;
        cron_mode)
                cron_mode
                ;;
        check_all)
                check_status
                tnsping_status
                archive_log_count
                archive_log_stb
                ;;
        *)
                echo "Usage: $0 {cron_mode|status|tnsping|archive_log|archive_log_stb|send_files|recover_remote_standby|recover_local_standby|switch_logfile_local|check_all}"
                exit 1
esac

New monitoring for standard environments

From some time now we have started using a “monitoring distribution”.
It uses passive monitoring where it can be used, and has lots of plugins for standard services.

It gives you a nice output, check it out:

Selection_045

Selection_044

Credit has to be given to http://mathias-kettner.de/
All you need to know is here => http://omdistro.org/

To install the FULL pack you just need to do the following:


# where to start => http://omdistro.org/

wget http://ftp.cica.es/epel/6/x86_64/epel-release-6-8.noarch.rpm
rpm -ivh epel-release-6-8.noarch.rpm
rpm -Uvh "https://labs.consol.de/repo/stable/rhel6/x86_64/labs-consol-stable.rhel6.noarch.rpm"
yum update
yum install omd-1.10
omd create monit
yum install check-mk-agent
omd create lab

After this, you can go to your server http://server/monit/ with user omdadmin and password omd , and you will find yourself adding machines to the monitoring.

The agents can be found on the server itself, at the /opt/omd/sites/monit/share/agents/ directory (I may misspell but is arround that )

Visual Basic script to pack, zip and send a full report.

Each first Monday of each month I needed to prepare and send a report to some customers.

Everything was on windows, so Visual Basic was the one to be used.
Nowadays I would ask to reinstall the machine to be able to use Powershellv3 or so, but at that point this is what I got.

'compress folder and send a zip file
'TODO : 


Option Explicit

Dim outputFileZip, outputErrFile, mailTextFile, reportFolder, emailFrom, emailTo, dateStamp, emailSubject
outputFileZip = "D:\Omv_Reports\OmniVisionGQR-ReportLastMonth.zip"
mailTextFile = "D:\Omv_Reports\report_scripts\file1.txt"
outputErrFile = "D:\Omv_Reports\report_scripts\file2.txt"
reportFolder = "D:\Omv_Reports\OmniVisionGQR-LastMonth"
emailFrom = "edited@edited.es"
emailTo = "edited@edited.es;edited@edited.es;edited@edited.es"
emailSubject = "Summary report for the A3S "
dateStamp = Date()

' -=-=-=-=-=-=-=-=-=-
''''''''''''''' Prepare log file
dim loggit_logfilename, loggit_fso, loggit_silent, tempFile

loggit_logfilename = outputErrFile
loggit_silent = true   ' log file only or with MsgBox/Echo
set loggit_fso = CreateObject("Scripting.FileSystemObject")
set tempFile = loggit_fso.OpenTextFile(loggit_logfilename, 2, True)
tempFile.Write ""
tempFile.Close 

' -=-=-=-=-=-=-=-=-=-
sub loggit (msg)
	Dim stream
    set stream = loggit_fso.OpenTextFile(loggit_logfilename, 8, True)
    stream.writeline date & " " & time & ": " & msg
    stream.close
    if not loggit_silent then
       WScript.echo msg 
    end if   
end sub
' -=-=-=-=-=-=-=-=-=-

'''''''''''''''''''' Log file ready. :)
loggit "Started..."

' -=-=-=-=-=-=-=-=-=-
' Funtion used on date string
' -=-=-=-=-=-=-=-=-=-

Dim objFSO,strDate

Function padDate(intNumber)
	if intNumber <= 9 Then
		padDate = "0" & CStr(intNumber)
	Else
		padDate = CStr(intNumber)
	End If
End Function

' -=-=-=-=-=-=-=-=-=-


'''''''''''''''beggin
Dim arrResult
'''ZipFolder funcion is at the end of this file. 
loggit "Launch ziping function."
arrResult = ZipFolder( reportFolder, outputFileZip )
loggit "Finish ziping function. "

If arrResult(0) = 0 Then
    If arrResult(1) = 1 Then
       'WScript.Echo "Done; 1 empty subfolder was skipped."
	   loggit "Done; 1 empty subfolder was skipped."
    Else
       'WScript.Echo "Done; " & arrResult(1) & " empty subfolders were skipped."
	   loggit "Done; " & arrResult(1) & " empty subfolders were skipped."
    End If
Else
    'WScript.Echo "ERROR ziping the lastMonth folder. Call Marc Riera ASAP" & Join( arrResult, vbCrLf )
	loggit "ERROR ziping the lastMonth folder. We are going to look at this and report back to you as soon as possible. :: " & Join( arrResult, vbCrLf )
End If

'''' Writing Done before the file gets into the may body text.
loggit "Done."
''''
'prepare mail content
Dim objEmail
Set objEmail = CreateObject("CDO.Message")
Const ForReading = 1, ForWriting = 2, ForAppending = 8
Dim fso, f, f2
Set fso = CreateObject("Scripting.FileSystemObject")
'Open the file for reading
Set f = fso.OpenTextFile(mailTextFile, ForReading)
Set f2 = fso.OpenTextFile(outputErrFile, ForReading)
'The ReadAll method reads the entire file into the variable BodyText
Dim BodyText
BodyText = f.ReadAll&f2.ReadAll
'Close the file
f.Close
f2.Close
Set f = Nothing
Set f2 = Nothing


'send mail
objEmail.From = emailFrom
objEmail.To = emailTo
objEmail.Subject = emailSubject & " --- " & dateStamp
objEmail.TextBody = BodyText
objEmail.AddAttachment outputFileZip
objEmail.Configuration.Fields.Item _
    ("http://schemas.microsoft.com/cdo/configuration/sendusing") = 2
objEmail.Configuration.Fields.Item _
    ("http://schemas.microsoft.com/cdo/configuration/smtpserver") = _
        "172.16.23.135"
objEmail.Configuration.Fields.Item _
    ("http://schemas.microsoft.com/cdo/configuration/smtpserverport") = 25
objEmail.Configuration.Fields.Update
objEmail.Send

loggit "File has been send"

Set objFSO = CreateObject("Scripting.FileSystemObject")
'MsgBox	Year(Date) & "-" & padDate(Month(Date)) & "-" & padDate(Day(Date))
strDate = Year(Date) & "-" & padDate(Month(Date)) & "-" & padDate(Day(Date))

loggit "Moving zip file to zip file with date"
If (objFSO.FileExists(outputFileZip)) Then
	objFSO.MoveFile outputFileZip , outputFileZip & "-" & strdate & ".zip"
End If
loggit "Moved."
loggit "Done. :) "

'''
Function ZipFolder( myFolder, myZipFile )
' This function recursively ZIPs an entire folder into a single ZIP file,
' using only Windows' built-in ("native") objects and methods.
'
' Last Modified:
' March 8, 2012
'
' Arguments:
' myFolder   [string]  the fully qualified path of the folder to be ZIPped
' myZipFile  [string]  the fully qualified path of the target ZIP file
'
' Return Code:
' An array with the error number at index 0, the source at index 1, and
' the description at index 2. If the error number equals 0, all went well
' and at index 1 the number of skipped empty subfolders can be found.
'
' Notes:
' [1] If the specified ZIP file exists, it will be overwritten
'     (NOT APPENDED) without notice!
' [2] Empty subfolders in the specified source folder will be skipped
'     without notice; lower level subfolders WILL be added, wether
'     empty or not.
'
' Based on a VBA script (http://www.rondebruin.nl/windowsxpzip.htm)
' by Ron de Bruin, http://www.rondebruin.nl
'
' (Re)written by Rob van der Woude
' http://www.robvanderwoude.com
' (Re) Joan Marc Riera - Bull - added outfile for errors/log

    ' Standard housekeeping
    Dim intSkipped, intSrcItems
    Dim objApp, objFolder, objFSO, objItem, objTxt, objErrFSO, objErrTxt
    Dim strSkipped

    Const ForWriting = 2

    intSkipped = 0
	
    ' Make sure the path ends with a backslash
    If Right( myFolder, 1 ) <> "\" Then
        myFolder = myFolder & "\"
	Else
		loggit "	- The path does  not end with backslash " & "\" 
    End If

    ' Use custom error handling
    On Error Resume Next

    ' Create an empty ZIP file
    Set objFSO = CreateObject( "Scripting.FileSystemObject" )
    Set objTxt = objFSO.OpenTextFile( myZipFile, ForWriting, True )
    objTxt.Write "PK" & Chr(5) & Chr(6) & String( 18, Chr(0) )
    objTxt.Close
    Set objTxt = Nothing

    ' Abort on errors
    If Err Then
        ZipFolder = Array( Err.Number, Err.Source, Err.Description )
		loggit "	- ERROR !!!! - " & Err.Number & " " & Err.Source & " " & Err.Description & "------" 
        Err.Clear
        On Error Goto 0
        Exit Function
    End If
    
    ' Create a Shell object
    Set objApp = CreateObject( "Shell.Application" )
	
	loggit "	- copy files to compressed folder . started. "
    ' Copy the files to the compressed folder
    For Each objItem in objApp.NameSpace( myFolder ).Items
        If objItem.IsFolder Then
            ' Check if the subfolder is empty, and if
            ' so, skip it to prevent an error message
            Set objFolder = objFSO.GetFolder( objItem.Path )
            If objFolder.Files.Count + objFolder.SubFolders.Count = 0 Then
                intSkipped = intSkipped + 1
            Else
                objApp.NameSpace( myZipFile ).CopyHere objItem
            End If
        Else
            objApp.NameSpace( myZipFile ).CopyHere objItem
        End If
    Next
	loggit "	- copy files to compressed folder . finished ."
	
    Set objFolder = Nothing
    Set objFSO    = Nothing

    ' Abort on errors
    If Err Then
        ZipFolder = Array( Err.Number, Err.Source, Err.Description )
		loggit "	- ERROR !!!! - " & Err.Number & " " & Err.Source & " " & Err.Description & "------" 
        Set objApp = Nothing
        Err.Clear
        On Error Goto 0
        Exit Function
    End If
	loggit "	- Compression started.  "
    ' Keep script waiting until compression is done
    intSrcItems = objApp.NameSpace( myFolder  ).Items.Count
    Do Until objApp.NameSpace( myZipFile ).Items.Count + intSkipped = intSrcItems
        WScript.Sleep 2000
    Loop
    Set objApp = Nothing
	loggit "	- Compression finished.  "
	
    ' Abort on errors
    If Err Then
        ZipFolder = Array( Err.Number, Err.Source, Err.Description )
		loggit "	- ERROR - " & Err.Number & " " & Err.Source & " " & Err.Description & "------" 
        Err.Clear
        On Error Goto 0
        Exit Function
    End If

    ' Restore default error handling
    On Error Goto 0

    ' Return message if empty subfolders were skipped
    If intSkipped = 0 Then
        strSkipped = "No items skipped."
    Else
        strSkipped = "skipped empty subfolders"
    End If

    ' Return code 0 (no error occurred)
    ZipFolder = Array( 0, intSkipped, strSkipped )
End Function

' close error file, here is where loggit sends the strings
Set fso = Nothing

WScript.Quit(0)

Use python to handle email, translate , and publish to wordpress

This weekend I tried to refresh my python with one single idea in mind.

But now I don’t have time to explain it, so here comes the code:

#!/usr/bin/env python
#
#joanmarcrieraATgmail(dot)com

from GmailClass import GmailClass
import goslate
import datetime, xmlrpclib
import time

from wordpress_xmlrpc import Client, WordPressPost
from wordpress_xmlrpc.methods.posts import NewPost
from datetime import datetime


gs=goslate.Goslate()

correo=GmailClass("jmraccount@gmail.com","Password","Foldera")
correo.connect()
correo.select_mailbox()
no_leidos=correo.get_unreaded_list()

wp_url = 'http://www.joanmarcriera.es/blog/xmlrpc.php'
wp_username = 'username'
wp_password = 'password'




for un_correo_no_leido in no_leidos[0].split():
      
    print "\nProcessing one unread..."
    msg=correo.get_a_mail(un_correo_no_leido)
    titulo_original=correo.get_the_subject(msg)
    print "\t " + titulo_original
    body_original=correo.get_the_body(msg)
    titulo_es=gs.translate(titulo_original,'es')
    body_es=gs.translate(body_original,'es')
      
    title=titulo_es
    content=body_es
    
    post = WordPressPost()
    post.title = title
    post.content = content
    post.date = datetime(2014, 8, 10, 12, 34, 2, 860000)
    post.post_status = 'draft'
    try:
        wp = Client(wp_url,wp_username,wp_password) 
        wp.call(NewPost(post))
    except  ServerConnectionError as e:
        #do something with e
        #or by now pass
        print "serverconnection error"
        pass
    except ProtocolError as e:
        #do something or pass
        print "protocol error"
        pass
    else:
        del post
        del wp
    
    print "\t Done."
    print "Now wait 10 seconds\n"
    time.sleep(10)
    
#!/usr/bin/env python
#
#joanmarcrieraATgmail(dot)com

import sys
import imaplib
#not needed #import getpass
import email
import email.header
import datetime

#class to manage gmail.
class GmailClass(object):
   
    #Basic initialize.
    #Here is were we need to declare which account and folder we would like to process
    #example a=GmailClass('jmr@gmail.com','superpass','kate')
    def __init__(self, account, pwsd, folder):
        self.folder = folder
        self.email_account = account
        self.email_password = pwsd

    #automatic for test 
    def prepare(self):
        self.connect()
        self.select_mailbox()
        
    
    #Connect to the mailbox.
    def connect(self):
        self.mailbox = imaplib.IMAP4_SSL('imap.gmail.com')
        try:
            rv, data = self.mailbox.login(self.email_account, self.email_password)
        except imaplib.IMAP4.error:
            print "LOGIN FAILED!!! "
            sys.exit(1)
        print rv, data
        print "You can now select a mailbox"
    
    #just if you want to know how many folders do you have    
    def list_mailboxes(self):
        #needs pretty print
        rv, mailboxes = self.mailbox.list()
        if rv == 'OK':
            print "Mailboxes:"
            print mailboxes
            print "Select one of them to be able to process it."
    
    #go into the folder
    def select_mailbox(self):
        rv, data = self.mailbox.select(self.folder)
        if rv == 'OK':
            print "You can now start processing mailbox...\n"
        else:
            print "ERROR: Unable to open mailbox ", rv    
            
    #which mails are still unread
    def get_unreaded_list(self):
        rv, data = self.mailbox.search(None, "UnSeen")
        if rv != 'OK':
            print "No messages found!"
            return
        else:
            print "Messages found "
            return data
    
    #return a mail object
    #the list from get_unreaded_list needs to be looped data[i]
    def get_a_mail(self,num):
        rv, data = self.mailbox.fetch(num, '(RFC822)')
        if rv != 'OK':
            print "ERROR getting message", num
            return
        else:
            msg = email.message_from_string(data[0][1])
            return msg

     
    #from a mail object get the subject   
    def get_the_subject(self,msg):
        decode = email.header.decode_header(msg['Subject'])[0]
        subject = unicode(decode[0])
        #print 'Message Subject: %s' % ( subject)
        return subject
    
    #from a mail object the date
    def get_the_date(self,msg):
        #print 'Raw Date:', msg['Date']
        return msg['Date']

        
    #from a mail object get the body    
    def get_the_body(self,msg):
        maintype = msg.get_content_maintype()
        if maintype == 'multipart':
            for part in msg.get_payload():
                if part.get_content_maintype() == 'text':
                    return part.get_payload()
        elif maintype == 'text':
            return msg.get_payload()


    #method to process mailbox    
    def process_mailbox(self):
        """
        Do something with emails messages in the folder.  
        For the sake of this example, print some headers.
        """
        rv, data = self.mailbox.search(None, "ALL")
        if rv != 'OK':
            print "No messages found!"
            return
    
        for num in data[0].split():
            rv, data = self.mailbox.fetch(num, '(RFC822)')
            if rv != 'OK':
                print "ERROR getting message", num
                return
    
            msg = email.message_from_string(data[0][1])
            decode = email.header.decode_header(msg['Subject'])[0]
            subject = unicode(decode[0])
            print 'Message %s: %s' % (num, subject)
            print 'Raw Date:', msg['Date']
            # Now convert to local date-time
            date_tuple = email.utils.parsedate_tz(msg['Date'])
            if date_tuple:
                local_date = datetime.datetime.fromtimestamp(
                    email.utils.mktime_tz(date_tuple))
                print "Local Date:", \
                    local_date.strftime("%a, %d %b %Y %H:%M:%S")
                
    #Close the sessions at the end
    def close(self):
        self.mailbox.close()
        self.mailbox.logout()