Tema 4.5 - Aplicación Web Log Analyzer

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24

//calculates size of log, and provides min, max and average size
// caching is done for repeatedly used factors
def calcContentSize(log: RDD[AccessLogs]) = {
 val size = log.map(log => log.contentSize).cache()
 val average = size.reduce(_ + _) / size.count()
 println("ContentSize:: Average :: " + average + " " +
 " || Maximum :: " + size.max() + " || Minimum ::" + size.min() )
 }
//Send all the response code with its frequency of occurrence as Output
def responseCodeCount(log: RDD[AccessLogs]) = {
 val responseCount = log.map(log => (log.responseCode, 1))
 .reduceByKey(_ + _)
 .take(1000)
 println(s"""ResponseCodes Count : ${responseCount.mkString("[", ",", "]")} """)
 }
//filters ipaddresses that have more then 10 requests in server log
 def ipAddressFilter(log: RDD[AccessLogs]) = {
 val result = log.map(log => (log.ipAddr, 1))
 .reduceByKey(_ + _)
 .filter(count => count._2 > 1)
 // .map(_._1).take(10)
 .collect()
 println("IP Addresses Count :: ${result.mkString("[", ",", "]")}" )
 } }