#!/usr/bin/perl -w
#
# Copyright (C) 2012, 2013, 2014 Mark Holler/Camalie Networks LLC
#
# pcDpush.pl is a PERL program that runs on a pcDuino-Nano to push data from
# the pcDuino-Nano or attached arduino shields to a Camalie Networks open data
# web server. HTTP protocol is used to push the data which is then stored on
# the server in an RRDTools database. An open source program, drraw.cgi,
# running on the server can then be used to access graphical trend charts of
# the data and/or csv files with the data which can be imported into a spread
# sheet. Open Data can be seen by other users but, cannot be deleted by them.
# NOTE: This program when installed along with a shell script to initialize it
# sends data as soon as the device it is installed on is connected to the internet
# and powered up. This program includes a command interpreter suitable for
# executing commands returned to it by the server it has pushed data to.
# pcDpush.pl is free software: you can redistribute it and/or modify
# it under the terms of the GNU General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
# see . http://www.gnu.org/licenses/gpl.html
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# 2. All advertising materials mentioning features or use of this software
# must display the following acknowledgement:
# This product includes software developed by Camalie Networks.
# 3. The name of the author may not be used to endorse or promote products
# derived from this software without specific prior written permission.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
# IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
# OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
# IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED
# TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE,
# EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
# $Id: pcDpush.pl 10/23/2014 12:15:15:15 holler $
#
# Home Page -- http://camalienetworks.com
# Notes:
# Must install vnstat network monitor for gateway monitoring
# Must install cpu_id.ko in /home/ubuntu/cs3code to get cpuid
use LWP::Simple;
use threads ('stack_size' => 64*4096,'yield', 'exit' => 'threads_only','stringify');
use threads::shared;
use Thread::Queue;
#use strict;
my $networkType = "Camalie Networks Data Pusher for pcDuino-Nano ";
my $codeRevision = "pcDpush.pl 10/24/14 M.H.";
#************ Push Destination Server(s) **********************************************************
my @cloudServers = ("http://camalie.net/cs3/snetServe3.cgi"); # Single push for less verbosity during Dev.
#my @cloudServers = ( "http://107.21.101.71/cs3"); # Aji's new non Amazon server
#my @cloudServers = (); # No push during development to avoid sending trash to servers
#my @cloudServers = ("http://camalie.net/cs3/snetServe3.cgi", "http://yo-z-o.com/snetServe3.cgi", "http://camalienetworks.net/cs3/snetServe3.cgi");
# The remote server must be set up with RRDTools and have a snetServe3.cgi script installed.
# It must also have sqlite installed and PERL modules DBI and DBD-sqlite installed.
# The User interface is a PERL script called CS3Viewxxx.cgi
# snetServe3.cgi is a CGI script that Apache runs when it is invoked via a port 80 request from this program.
my $touchFile="/home/ubuntu/cs3code/pushed";
# Maximum working threads ****************
my $MAX_THREADS = 10;
my $pushCount = 0; # how many packets pushed.
my $samplingInterval = 20; # Time between samples pushed in seconds
my $gtwSensorType=208; # sensor type given to data packet from the gateway.
my $gtw_pcDnSensorType=209; # sensor type given to data packet from a pcDuino3-Nano gateway.
my $dummyVar = 0;
#********************* SETUP ******************************************************************************************
print "$networkType \n";
print "$codeRevision \n";
#************** Get the CPU ID ********************************************************
if(!open(CPUID, '/proc/cpuid')) { #eliminates an error message if cpuid already extracted
system("insmod /home/ubuntu/cs3code/cpu_id.ko");
}
open(CPUID, '/proc/cpuid');
my $CPUid = 666; #Give $CPUid a default recognizable ID in case getting CPUid fails.
my $Rev = ;
print "\n$Rev";
$CPUid = ;
print "cpuid =$CPUid";
$CPUid = substr($CPUid, 3,32); #strip off ID: and take 32 characters leaving as character 33 is a newline
print "cpuid substring Used =$CPUid \n";
close(CPUID);
my $shieldID = 0; #0 for data coming from the Linux CPU itself- status data.
#1 for data coming from an arduino shield plugged into the pcDuino3-Nano
#X other numbers for other shields.
my %pktID = ( # This hash stores the elements which identify a specific data source
'net', $CPUid, # Three Letter Acronym for the network
'node', "",
'port', "",
'sensorType', "",
'RRDname', "" ); #fully path qualified file name of the RRD
#************** Get the local IP address of this CS3 gateway **************************************
#
system("ip addr show eth0 > localIPaddr"); # This gets a paragraph with the local IP address in it
open LOCALIPADDR, localIPaddr ;
read LOCALIPADDR, my $localIPaddrStr, 300;
#print "\n\nMy Local IP string = $localIPaddrStr\n";
my $ipAddrPos = index $localIPaddrStr, "inet ";
my $localIPaddr = substr $localIPaddrStr, ($ipAddrPos+5),15;
my $slashPos = index $localIPaddr, "/";
$localIPaddr = substr $localIPaddr, 0, $slashPos;
print "\nThe Local IP address of this CS3 gateway is $localIPaddr\n\n";
#********* Create the Idle Threads Queue and one Queue per thread ***********************************
# Threads add their ID to the IDLE_QUEUE when they are ready for work
my $IDLE_QUEUE = Thread::Queue->new();
# Thread work queues referenced by thread ID
my %work_queues;
# Create the thread pool
for (1..$MAX_THREADS) {
# Create a work queue for a thread
my $work_q = Thread::Queue->new();
# Create the thread, and give it the work queue
my $thr = threads->create('worker', $work_q);
# Remember the thread's work queue
$work_queues{$thr->tid()} = $work_q;
} # end creation of worker threads
#**************** MAIN LOOP STARTS HERE *******************************************************
while(1) {
#*********** Send gateway status packet to cloud ***************
sendStatus();
select(undef,undef,undef, $samplingInterval); # delay between samples
} # End MAIN LOOP
#***********************************************************************************************
#********************** Subroutine Implementations ********************************************
#***********************************************************************************************
#
#******* Send Gateway status data packet to cloud servers.
#
sub sendStatus {
print "\nSampling and Sending Gateway status\n";
my $unixTime= time;
my $urlString = ("?packetType="."data"."&netID=".$CPUid."&nodeID=".$shieldID."&port=gtw"."&sensorType=".$gtw_pcDnSensorType."&atTime=".$unixTime);
my $memFreekB;
my $loadAvg1min;
my $cpuUser;
my $cpuSys;
my $diskFreeGB;
my $netBWavg;
my $top = `top -b -n 1`; #get top information string
#print "$top\n";
my $posMf = index($top, 'free');
#print "Position of searchkey = $posMf\n";
#my $lengthNat = length($top);
#print "topStr length $lengthNat\n";
$memFreekB = substr($top,($posMf-25),7);
print "Free memory =$memFreekB\n";
my $position = index($top, 'average:');
$loadAvg1min = substr($top, ($position+8), 5);
print "Load Average 1 min =$loadAvg1min\n";
$position = index($top, '%us,');
$cpuUser = substr($top, ($position-4), 4);
print "CPU User =$cpuUser\n";
$position = index($top, '%sy,');
$cpuSys = substr($top, ($position-4), 4);
print "CPU System =$cpuSys\n";
my $df = `df`; #get df information string
#print "$df\n";
$position = index($df, '% /');
$diskFreeGB = substr($df, ($position-2), 2);# changed 10/15/14 M.H.
print "Disk Free % = $diskFreeGB\n";
my $vnstat = `vnstat`; #get vnstat information string
#print "$vnstat\n";
$position = index($vnstat, 'today');
$netBWavg = substr($vnstat, ($position+40), 5);
if ($netBWavg eq "x ") {
$netBWavg = 0;
print "Subsbstituting 0 for x in netBWavg. \n";
} # little hack to make output numeric before first valid average
print "Net Bandwidth 1D =$netBWavg\n";
#foreach $key (keys %ENV) {
# print "$key->$ENV{$key}\n";
#}
print "\nPush Count = $pushCount, free Memory = $memFreekB kB, Load Average 1 min = $loadAvg1min, CPU User = $cpuUser, CPU System = $cpuSys, Disk Free = $diskFreeGB, Net Bandwidth 1D = $netBWavg\n";
$urlString =($urlString."&pushCount=".$pushCount."&freeMem=".$memFreekB."&loadAvg1min=".$loadAvg1min."&cpuUser=".$cpuUser."&cpuSys=".$cpuSys."&diskFree=".$diskFreeGB."&netBWavg1D=".$netBWavg); # concatenate parameter onto end of http packet
#Strip out spaces from the urlString here.
while (($pos_1 = index($urlString,' ')) gt 0) {
#print "Index of first space = $pos_1 \n";
substr($urlString, $pos_1,1) = '';
}
#print " urlString with nulls/spaces stripped = $urlString|\n";
print "cs3 gateway Status packet sensorType $gtw_pcDnSensorType = \n $urlString \n";
# Push data to remote server via http: request.
pushHTTP("$urlString");
system("touch $touchFile");
#print "touched $touchFile" ;
$pushCount++;
}# end sub sendStatus()
#******** Request a command from each server and if command, execute ***************************
#
sub doCommands {
foreach $server (@cloudServers){
#print "Getting Command From: $server \n";
#***** Send Request to server for a command and execute
my $commandStr = get($server."?packetType=commandReq&netID=$CPUid");
print "Received = $commandStr \n";
my $destL=""; # need to clear this here.
my $paramR=""; # this too.
my $value="";
if(defined($commandStr)) { #if server online and responded, maybe with null command
my $commandHashRef = parseCommand($commandStr);
if(($commandHashRef)) { #if command parser returned a command hash
my %commandHash = %$commandHashRef; # dereference command hash ref to get command hash
#foreach $commandKey (keys %commandHash) {
#print "Command Hash $commandKey, $commandHash{$commandKey} "; # if ($commandHash{$commandKey});
#}
$paramR = $commandHash{command}; # should do some validity checking here.
$destL = $commandHash{nodeID};
$cmdTime = $commandHash{cmdTime}; $cmdTime = $cmdTime + 0; # to get rid of used once error msg.
$value = $commandHash{value};
if ($value) { # if value was provided with command
# end if command value defined
}elsif (!$commandHash{value} && $commandHash{command}) {
#command but no parameter passed with the command means read parameter value.
} # end elsif() read value of param and push to server
} # end if commandHash defined - non null command values received from server
}# end if(defined($commandStr) command string received from cloud server
} # end foreach $server
} # end doCommands()
#************************ Arduino Commands *******************************
# This is an example of some command strings that could be used to control an arduino shield plugged into the Nano.
# //
# // -ABC123--123XYZ arbitrary command--value pair
# // -ap3h set arduino pin PD3 High
# // -ap3 query arduino pin 3 state, mode and value
# // -ap4l set arduino pin 4 LOW until command to contrary, puts pin in digitarl out mode.
# // -ap5h--100 set arduino pin PD5 High for 100 seconds
# // -aps3 apply voltage to SS3 pin opt01/pwr
# // -rs4--300 apply current to coil of relay #4 on arduino relay shield
# // -rss3--180 apply current to coil of relay on SS3
# // -txt--Hello World Display text on arduino display shield.
# // -cp2--9 chargepump capacitor on SS2 to 9V and maintain until discharge command. or default timout
# // -av3--600 pulse bistable valve connected to SS3 open and after 10 minutes pulse it closed.
# // -sac1 sample the voltage on SS1 cap
# // -saa3 sample the voltage on pin A3 and send back to gateway.
# // -ps--23 play sound sample 23 on wav shield.
# // -ci--4 capture image with camera shield resolution 4 and send back.
#
#
#*********************************************************************************************
#********************** Parse commands from remote server ************************************
sub parseCommand {# receives a string and returns a hashRef which points to a Hash with the following keys
#print " parseCommand() received the following string $_[0] \n";
my $command = getValue($_[0],'command');
my $value = getValue($_[0],'value');# This will be null for a query command
my $node = getValue($_[0],'nodeID');
my $cmdTime = getValue($_[0],'cmdTime');
if (!($command eq "")) {
print " Command parsed= command $command, value $value, node $node cmdTime $cmdTime\n";
my $commandHash = {command=>$command, value=>$value, nodeID=>$node, cmdTime=>$cmdTime};
return $commandHash; # returns a reference to a Hash
}else {
#print "Command Parser did not receive a string to parse\n";
return "";
}
}
#********************** Push HTTP format data string into an available thread's queue *******************************
# Takes an http parameter string and sends it to the next available thread, The thread is responsible for pushing
# the data to all servers on the server list.
sub pushHTTP {
my $postString = $_[0];
# Wait for an available thread. idle thread IDs are put into the IDLE_QUEUE which is separate from the thread queues.
my $tid = $IDLE_QUEUE->dequeue(); # this better not block very often but it will if no thread IDs enqueued available.
# use dequeue_nb() "non blocking to avoid blocking if this becomes a problem.
# Give the thread some work to do; enqueue a postString for push
#my $work = $postString;
print "Enqueueing postString $postString \n";
$work_queues{$tid}->enqueue($postString); #put the $work item into the work queue associated with thread, $tid
# 5 lines below useless. the threads are always running. Left here for reference.
#my @threadsList = threads->list();
#my $thread_count = threads->list();
#my @running = threads->list(threads::running);
#my @joinable = threads->list(threads::joinable);
#print "Threads not currently detached = @threadsList Thread_count = $thread_count Running = @running Joinable = @joinable\n";
}
#********************** Push data to cloud servers *********************************************
# Takes an http parameter string and sends it to the list of cloud servers
# prints the response string from each server
# This function is called by the threads.
sub pushHTTPthread {
my $postString = $_[0];
my $server;
foreach $server (@cloudServers){
#print "Pushing to: $server $postString\n";
getprint($server.$postString);# Use this to see server's response.
#get($server.$postString); # Use this if you don't want to waste time printing to console.
}
}
#******************** worker thread subroutine, used to push HTTP format packets to servers *******************************
# MAX_THREADS copies of which run continuously after startup of this program
# Threads used in pushing to avoid full process hang when a server is slow or fails to respond to a push
#
sub worker
{
my ($work_q) = @_; #redundant use of a variable here creating mass confusion. ($work_q) here it is a list of the parameters passed to sub worker.
# it contains a Thread::Queue object in this case
# This thread's ID
my $tid = threads->tid();
# Work loop
do {
# Indicate to main that we are ready to do work and which thread we are. main will then enqueue a task in the queue associated with this thread
# but, only ever one since this subroutine doesn't get called until main see's it is idle
printf("Idle -> %2d\n", $tid);
$IDLE_QUEUE->enqueue($tid); #put our ID into the idle_queue
# could skip the idle queue and just sample each of the threads in sequence to see if they are still running or not.
# Wait for work from the queue This can block because it is in a separate thread Main thread needs to keep running.
my $work = $work_q->dequeue(); # again same variable reused in different scope. This code sucks.
# pull a work item off of the queue for this thread. This is stupid because only one item at a time ever get's put into a thread's queue
# $work here would be a list item capable of fitting into the queue list, it would have to be an array with the server name and packet string to push.
# If no more work, exit
#last if ($work < 0);
# Do some work while monitoring $TERM
printf(" %2d <- Working\n", $tid);
#while (($work > 0) && ! $TERM) {
print "thread $tid pushing $work \n\n";
#pushHTTP ("?packetType=data&netID=20501&nodeID=1084807054&port=0&sensorType=200&atTime=1401886521&an0=797&an1=1023&an2=1023&an3=1023");
pushHTTPthread ($work);
sleep (8);
#print "work = $work \n";
#$work -= sleep($work); # count down work until less than zero, dummy work
#}
# Loop back to idle state if not told to terminate
#last;
$dummyVar = $TERM; #getting rid of error msg..
} while (! $TERM);
# All done
printf("Finished -> %2d\n", $tid);
} # end subroutine worker()