satishgaudo.com

Ganglia : monitoring framework

Ganglia is a monitoring framework for clusters of servers. It records many statistics and can record custom defined ones too. It works in a distributed manner, with each machine you wish to collect statistics for running the Ganglia monitor deamon, gmond. Each monitoring deamon’s statistics are collected by a metadata daemon, gmetad, running on either one of the monitored hosts or a separate machine. Ganglia provides a PHP frontend which displays the data from gmetad in the form of pretty graphs.

For extra details: http://www.ibm.com/developerworks/wikis/display/WikiPtype/ganglia

On installation ganglia will show default metrics like cluster load, cpu status etc.

Admin can define metrics that he want using “gmetric” command like

/usr/bin/gmetric  -u hits -tfloat -n apache_status_200  -v 100

where n is the metric name and v is the value

Following are the PHP scripts called on crontab,

1. Apache Access log metric to count hits status:

$gmetric_command = “/usr/bin/gmetric”;
//cat access.log | grep “31/Jul/2010:09″ | cut -f9 -d ” ” | sort | uniq -c
$sApacheLogFile=APACHE_LOG_DIR.”access.log”;
$sCurrentDate=date(”d/M/Y:H”);
$sStatCommand=”cat “.$sApacheLogFile.” | grep ‘”.$sCurrentDate.”‘ | cut -f9 -d ‘ ‘ | sort | uniq -c”;
$aOutput=Array();
$sOp=exec($sStatCommand,$aOutput);
if(is_array($aOutput)){
foreach($aOutput as $iK =>$sValue) {
$aData=Array();
$aData=explode(” “,trim($sValue));
if($aData[0]>0){
system($gmetric_command . ” -u hits -tfloat -n apache_status_”.$aData[1].” -v ” . $aData[0]);
}
}
}

2.  Mysql status metrics:

//gmetric command
$stats_command = “/usr/bin/mysqladmin -u root –password=mAnG0pp1 extended-status”;
$gmetric_command = “/usr/bin/gmetric”;
$aCounterMetrics = Array(
“Bytes_received” => “bytes”,
“Bytes_sent” => “bytes”,
“Com_delete” => “operations”,
“Com_insert” => “operations”,
“Com_replace” => “operations”,
“Com_select” => “operations”,
“Com_update” => “operations”,
“Key_reads” => “operations”,
“Qcache_hits” => “hits”,
“Questions” => “queries”,
“Connections” => “connections”,
“Threads_created” => “threads”,
“Slow_queries” => “queries”
);
$aAbsoluteMetrics = array(
“Threads_connected” => “threads”,
“Threads_running” => “threads”
);
$sMysqlLogFile=LOG_DIR.”mysql_stats.txt”;

if(!file_exists($sMysqlLogFile)){
//if no file , create a one
system(”$stats_command > $sMysqlLogFile”);
}else {
//get the old stats data
$aStatsOld=file($sMysqlLogFile);
$aMyStatsOld=Array();
$i=0;
if(is_array($aStatsOld)&& count($aStatsOld)>0){
for($i=3;$i<(count($aStatsOld)-1);$i++){
$aStatTemp=explode(”|”,$aStatsOld[$i]);
$aMyStatsOld[trim($aStatTemp[1])]=trim($aStatTemp[2]);
}
}
unset($aStatTemp);
//get the time file modified last time
$sFileModTime=filemtime($sMysqlLogFile);
unlink($sMysqlLogFile);
//get the current stats
system(”$stats_command > $sMysqlLogFile”);
$sCurrentTime=time();
$aStatsNew=file($sMysqlLogFile);
$aMyStatsNew=Array();
if(is_array($aStatsNew)&& count($aStatsNew)>0){
for($i=3;$i<(count($aStatsNew)-1);$i++){
$aStatTemp=explode(”|”,$aStatsNew[$i]);
$aMyStatsNew[trim($aStatTemp[1])]=trim($aStatTemp[2]);
}
}
$iTimeDiff=$sCurrentTime-$sFileModTime;
if($iTimeDiff<1){
die(”Time difference can’t be less than 1″);
}
# Calculate deltas for counter metrics and send them to ganglia
foreach($aCounterMetrics as $sMetric =>$sUnit){
//print $aMyStatsNew[$sMetric].” – “.$aMyStatsOld[$sMetric].”===<br>”;
$iRate = ($aMyStatsNew[$sMetric] – $aMyStatsOld[$sMetric]) / $iTimeDiff;

if ($iRate>=0) {
system($gmetric_command . ” -u ‘$sUnit/sec’ -tfloat -n mysqld_” . $sMetric . ” -v ” . $iRate);
}
}
//Just send absolute metrics. No need to calculate delta
foreach($aAbsoluteMetrics as $sMetric =>$sUnit){
system($gmetric_command . ” -u $sUnit -tuint16 -n mysqld_” .$sMetric. ” -v ” .$aMyStats[$sMetric]);
}
}

3.  Memcache status:

require_once(CLASS_PATH.’memcache.class.php’);
$oCache = new Cache();
$gmetric_command = “/usr/bin/gmetric”;
$sMemcacheLogFile=LOG_DIR.”memcache_stats.txt”;
$aOldStats=Array();
//get the memcache stats
$aNewStats=$oCache->getExtendedStats();
if(!file_exists($sMemcacheLogFile)){
$f=fopen($sMemcacheLogFile,”w”);
fwrite($f,serialize($aNewStats));
fclose($f);
}else {
$aOldStats=unserialize(file_get_contents($sMemcacheLogFile));
$sFileModTime=filemtime($sMemcacheLogFile);
$iTimeDiff=time()-$sFileModTime;

$f=fopen($sMemcacheLogFile,”w”);
fwrite($f,serialize($aNewStats));
fclose($f);
}

#################################################################################
# Memcache Hit Ratio
#################################################################################
foreach($aNewStats as $sHost =>$aData){
$sDisplayHost=$sHost;
$sDisplayHost=str_replace(”.”,”_”,$sDisplayHost);
$sDisplayHost=str_replace(”:”,”_”,$sDisplayHost);
$iHits   = $aData['get_hits'];
$iMisses = $aData['get_misses'];
$iTotalReq = $iHits + $iMisses;
$iHitRatio=0;
if ($iTotalReq == 0 ) {
$iHitRatio = 0.0;
} else {
$iHitRatio = $iHits / $iTotalReq;
}

system($gmetric_command . ” -u ratio -tfloat -n memcache_hit_ratio”.$sDisplayHost.” -v ” . $iHitRatio);
#################################################################################
# Calculate Memcache Fill Ratio
#################################################################################
$iTotalBytes = 0;
$iUsedBytes  = 0;
$iCurrConnections = 0;
$iCurrItems = 0;
$iFillRatio = 0;
$iEvictions = 0;
$iBytesRead = 0;
$iBytesWritten = 0;
$iTotalBytes = $aData['limit_maxbytes'];
$iUsedBytes  = $aData['bytes'];
$iFillRatio = $iUsedBytes/$iTotalBytes;
$iCurrConnections = $aData['curr_connections'];
$iCurrItems = $aData['curr_items'];

$iEvictions = $aData['evictions'];
$iBytesRead = $aData['bytes_read'];
$iBytesWritten = $aData['bytes_written'];

system($gmetric_command . ” -u ratio -tfloat -n memcache_fill_ratio”.$sDisplayHost.” -v ” . $iFillRatio );
system($gmetric_command . ” -u connections -tfloat -n memcache_curr_connections”.$sDisplayHost.” -v ” . $iCurrConnections );
system($gmetric_command . ” -u items -tfloat -n memcache_curr_items”.$sDisplayHost.” -v ” . $iCurrItems );
//calculate counter metrics such as bytes in/out and evictions

if (isset($aOldStats[$sHost]) && $iBytesRead >= $aOldStats[$sHost]['bytes_read']) {
system($gmetric_command . ” -u number -tuint32 -n memcache_evictions”.$sDisplayHost.” -v ” . ($iEvictions – $aOldStats[$sHost]['evictions']) );
system($gmetric_command . ” -u bytes/s -tfloat -n memcache_bytes_read”.$sDisplayHost.” -v ” . ( ($iBytesRead – $aOldStats[$sHost]['bytes_read']) / $iTimeDiff)  );
system($gmetric_command . ” -u bytes/s -tfloat -n memcache_bytes_written”.$sDisplayHost.” -v ” . ( ($iBytesWritten – $aOldStats[$sHost]['bytes_written']) / $iTimeDiff ) );
}

}

—memcache.class.php

<?php
/**
*
* Setup:
*
edit the singleton() metod
and define the list of memcached servers in a 2-d array
in the format
array(
array(’192.168.0.1′=>’11211′),
array(’192.168.0.2′=>’11211′),
);
*
*
* Usage:
*
<?php
//include the class name
include (’memcache.class.php’);

//store the variable
Cache::set(’key’,'abc’);

//increment/decrement the integer value
Cache::increment(’key’);
Cache::decrement(’key’);

//fetch the value by it’s key
echo Cache::get(’key’);

//delete the data
echo Cache::delete(’key’);

//Clear the cache memory on all servers
Cache::flush();

Cache::replace() and Cache::add are implemented also.

More information can be obtained here:
http://www.danga.com/memcached/
http://www.php.net/memcache

*/
/** @ingroup classes
*  This is the Classes group
*  @{
*/
/**
* @brief class Cache The class makes it easier to work with memcached servers and provides hints in the IDE like Zend Studio
* @version 1
*
*/

class Cache {
/**
* Resources of the opend memcached connections
* @var array [memcache objects]
*/
var $mc_servers = array();
/**
* Quantity of servers used
* @var int
*/

var $servers = array(

array(’127.0.0.1′=>’11211′)
);

var $mc_servers_count=1;

/**
* Accepts the 2-d array with details of memcached servers
*
* @param array $servers
*/
function Cache(){

/*
for ($i = 0, $n = count($this->servers); $i < $n; ++$i){
( $con = memcache_connect(key($this->servers[$i]), current($this->servers[$i])) )&&
$this->mc_servers[] = $con;
}
$this->mc_servers_count = count($this->mc_servers);
if (!$this->mc_servers_count){
$this->mc_servers[0]=null;
}
*/
}

/**
* Get statistics of the server
*
* @param string $key
* @param mix $value
* @return bool
*/
function getExtendedStats() {
$num=0;
foreach ($this->servers[$num] as $host => $port) {
$memcache_obj = memcache_pconnect($host, $port);
}
if (!$memcache_obj) return false;
return $memcache_obj->getExtendedStats();
}

//class end
}

?>