Commit c19acfe2 authored by Zied SELLAMI's avatar Zied SELLAMI
Browse files

Initial commit

parents
.idea/
\ No newline at end of file
name := "burndownchart"
import sbt._
val sparkVersion = "2.2.0"
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-core" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion,
"org.apache.spark" %% "spark-yarn" % sparkVersion
)
\ No newline at end of file
sbt.version = 0.13.16
\ No newline at end of file
import java.io.{File, PrintWriter}
import Query.{getIssues, getLabels, getMilestone, getNotes}
import org.apache.spark.sql.SparkSession
object DataReader {
def writeFile(fileName: String, content: String)={
val pw = new PrintWriter(new File(fileName))
pw.write(content)
pw.close
}
def deleteTmpDirFiles(dataDir: String): Unit ={
val tmpDir = new File(dataDir)
if(!tmpDir.exists()){
tmpDir.mkdirs()
}else{
tmpDir.listFiles().map(_.delete())
}
}
def readAndSaveData(projectName: String, milestoneName: String, dataDir: String)(implicit privateToken: String, sparkSession: SparkSession) = {
import sparkSession.implicits._
deleteTmpDirFiles(dataDir)
val issuesPath = s"$dataDir/issues.json"
val milestonePath = s"$dataDir/milestone.json"
val labelsPath = s"$dataDir/labels.json"
val pName = projectName.replace("/","%2F")
val mName = milestoneName.replace(" ","+").replace("#","%23")
val issues = getIssues(pName, mName)
val milestone = getMilestone(pName)
val labels = getLabels(pName)
writeFile(issuesPath, issues)
writeFile(milestonePath, milestone)
writeFile(labelsPath, labels)
val issuesId = sparkSession.read.json(s"$dataDir/issues.json")
.as[Issue]
.map(_.iid.toString())
.collect().toSeq
val notes = getNotes(pName, issuesId)
notes.map(entry => writeFile(s"$dataDir/notes_${entry._1}.json", entry._2))
}
def getNotesPath(dataDir: String): Seq[String] = {
new java.io.File(dataDir)
.listFiles()
.filter(_.getName.contains("notes_"))
.map(_.getAbsolutePath)
.toSeq
}
}
//case class GraphValue(date: java.sql.Date, isWeekend: Boolean, value: Float)
object GraphBuilder {
private def toD3JSDate(stringDate: String): String = {
val dateElement = stringDate.split(", ").toSeq
dateElement.head +", "+ (dateElement.apply(1).toInt - 1) +", "+ dateElement.last
}
private def createD3JSEntryFromGraphValue(gv:GraphValue): String = {
s"{date: new Date(${toD3JSDate(gv.date.toString.replace("-",", "))}), points: ${gv.value}}"
}
def getHTMLGraphCode(idealGraphValues: Seq[GraphValue], actualGraphValues: Seq[GraphValue], title: String): String ={
val idealLine = idealGraphValues.map(gv => createD3JSEntryFromGraphValue(gv)).mkString(",\n")
val actualLine = actualGraphValues.map(gv => createD3JSEntryFromGraphValue(gv)).mkString(",\n")
val html = s"""<!DOCTYPE html>
<html>
<head>
<meta charset=utf-8 />
<title>${title}</title>
<style>
body {
background-color: #FFFFFF
}
.axis {
font: 10px sans-serif;
}
.axis path,
.axis line {
fill: none;
stroke: #D4D8DA;
stroke-width: 2px;
shape-rendering: crispEdges;
}
.line {
fill: none;
stroke: #6F257F;
stroke-width: 5px;
}
.overlay {
fill: none;
pointer-events: all;
}
.focus circle {
fill: #F1F3F3;
stroke: #6F257F;
stroke-width: 5px;
}
.hover-line {
stroke: #6F257F;
stroke-width: 2px;
stroke-dasharray: 3,3;
}
.line {
fill: none;
stroke-width: 1.5px;
}
.line.ideal {
stroke: steelblue;
}
.line.actual {
stroke: orange;
}
.grid .tick {
stroke: lightgrey;
opacity: 0.7;
}
.grid path {
stroke-width: 0;
}
</style>
<script src="http://d3js.org/d3.v3.min.js"></script>
</head>
<body>
<div id="chart2" style="height:1200px">
<svg></svg>
</div>
<script type="text/javascript">
var margin = {top: 20, right: 20, bottom: 30, left: 50},
width = 960 - margin.left - margin.right,
height = 500 - margin.top - margin.bottom;
var x = d3.time.scale()
.range([0, width]);
var y = d3.scale.linear()
.range([height, 0]);
var ideal = [
${idealLine}
];
var actual = [
${actualLine}
];
var idealLine = d3.svg.line()
.x(function(d) { return x(d.date); })
.y(function(d) { return y(d.points); });
var actualLine = d3.svg.line()
.x(function(d) { return x(d.date); })
.y(function(d) { return y(d.points); });
x.domain(d3.extent(ideal.concat(actual), function(d){return d.date;}));
y.domain(d3.extent(ideal.concat(actual), function(d){return d.points;}));
var xAxis = d3.svg.axis()
.scale(x)
.orient("bottom")
.tickFormat(d3.time.format("%b-%d"));
var yAxis = d3.svg.axis()
.scale(y)
.orient("left");
var chart = d3.select("#chart2 svg")
.attr("class", "chart")
.attr("width", width + margin.left + margin.right)
.attr("height", height + margin.top + margin.bottom)
.append("g")
.attr("transform", "translate(" + margin.left + "," + margin.top + ")");
// Create the x-axis
chart.append("g")
.attr("class", "x axis")
.attr("transform", "translate(0," + height + ")")
.call(xAxis);
// create Title
chart.append("text")
.attr("x", (width / 2))
.attr("y", 0 - (margin.top / 4))
.attr("text-anchor", "middle")
.style("font-size", "16px")
.style("text-decoration", "underline")
.text("${title}");
// Create the y-axis
chart.append("g")
.attr("class", "y axis")
.call(yAxis)
.append("text")
.attr("transform", "rotate(-90)")
.attr("y", 6)
.attr("dy", ".71em")
.style("text-anchor", "end")
.text("Points");
// Paint the ideal line
chart.append("path")
.datum(ideal)
.attr("class", "line ideal")
.attr("d", idealLine);
// Paint the actual line
chart.append("path")
.datum(actual)
.attr("class", "line actual")
.attr("d", actualLine);
function make_x_axis() {
return d3.svg.axis()
.scale(x)
.orient("bottom")
.ticks(${idealGraphValues.size})
}
function make_y_axis() {
return d3.svg.axis()
.scale(y)
.orient("left")
.ticks(10)
}
chart.append("g")
.attr("class", "grid")
.attr("transform", "translate(0," + height + ")")
.call(make_x_axis()
.tickSize(-height, 0, 0)
.tickFormat("")
)
chart.append("g")
.attr("class", "grid")
.call(make_y_axis()
.tickSize(-width, 0, 0)
.tickFormat("")
)
</script>
</body>
</html> """
html
}
}
import scala.io.Source
import java.net.{HttpURLConnection, URL}
object Query {
val readTimeout = 20000
val connectTimeout = 20000
private def get(url: String)(implicit privateToken: String): String = {
val connection = (new URL(url)).openConnection.asInstanceOf[HttpURLConnection]
connection.setConnectTimeout(connectTimeout)
connection.setReadTimeout(readTimeout)
connection.setRequestProperty("Accept","application/json")
connection.setRequestProperty("Content-Type","application/json")
connection.setRequestProperty("PRIVATE-TOKEN",s"$privateToken")
connection.setRequestMethod("GET")
val inputStream = connection.getInputStream
val content = Source.fromInputStream(inputStream).mkString
if (inputStream != null) inputStream.close
content
}
def getMilestone(projectName: String)(implicit privateToken: String): String ={
val url = s"https://ci.linagora.com/api/v4/projects/$projectName/milestones?per_page=500"
get(url)
}
def getIssues(projectName: String, milestoneName: String)(implicit privateToken: String): String ={
val url = s"https://ci.linagora.com/api/v4/projects/$projectName/issues?per_page=500&milestone=$milestoneName"
get(url)
}
def getLabels(projectName: String)(implicit privateToken: String): String ={
val url = s"https://ci.linagora.com/api/v4/projects/$projectName/labels?per_page=500"
get(url)
}
def getNotes(projectName: String, issuesIds: Seq[String])(implicit privateToken: String): Map[String, String] ={
val url = s"https://ci.linagora.com/api/v4/projects/$projectName/issues/ISSUE_ID/notes?per_page=500"
issuesIds.map(issueId => (issueId -> get(url.replace("ISSUE_ID",issueId)))).toMap
}
}
import java.sql.{Date, Timestamp}
import org.apache.spark.sql.functions._
import org.apache.spark.sql.{DataFrame, Dataset, SparkSession}
import scala.util.Try
//Case class for JSON Data
case class Milestone(id: Long, iid: Long, project_id: Long, due_date: Date, start_date: Date, title: String)
case class Assignee(state: String, id: Long, name: String, username: String)
case class Issue(project_id: Long, state: String, iid: Long, id: Long, assignee: Assignee, updated_at: Timestamp, created_at: Timestamp, due_date: Date, milestone: Milestone, labels: Seq[String])
case class Label(id: Long, name: String)
case class Note(noteable_id: Long, body: String, created_at: Timestamp, updated_at: Timestamp)
//Case class for building burndownchart
case class LabelScore(id: Long, score: Float)
case class Operation(id: Long, date: Date, timestamp: Timestamp, operation: String, value: Float){
}
case class GraphValue(date: Date, isWeekend: Boolean, value: Float)
object Runner {
val ADD = "ADD"
val REMOVE = "REMOVE"
val CLOSED = "CLOSED"
def getLabelsScore(labelsDF: Dataset[Label])(implicit sparkSession: SparkSession): Seq[LabelScore] ={
import sparkSession.implicits._
labelsDF.map(label => {
if(Try(label.name.toFloat).isSuccess){
LabelScore(label.id, label.name.toFloat)
}else{
LabelScore(label.id, -1f)
}
}).collect().toSeq
}
def prefixDataFrameColumns(df: DataFrame, prefix: String): DataFrame = {
val columnsName = df.columns.map(name=> s"${prefix}_${name}").toSeq
df.toDF(columnsName: _*)
}
def main(args: Array[String]) ={
require(args.length == 4, "USAGE: projectName milestoneName privateToken tmpDir")
implicit val sparkSession = SparkSession
.builder()
.appName("SparkSession for Burndownchart")
.master("local[*]")
.getOrCreate()
import sparkSession.implicits._
val projectName = args(0)
val milestoneName = args(1)
implicit val privateToken = args(2)
val dataDir = args(3) +"/burndownchart"
val title = s"Project Name: $projectName - Milestone: $milestoneName"
//val projectName = "LINAGORA/LGS/OpenPaaS/james"
//val milestoneName = "Sprint 30"
// val projectName = "LINAGORA/LGS/OpenPaaS/linagora.esn.admin"
// val milestoneName = "Sprint 9"
// val projectName = "LINAGORA/LGS/OpenPaaS/linagora.esn.calendar"
// val milestoneName = "Sprint 24"
// val projectName = "LINAGORA/LGS/OpenPaaS/linagora.esn.chat"
// val milestoneName = "Sprint 10"
// val projectName = "LINAGORA/LGS/OpenPaaS/saas-portal"
// val milestoneName = "Sprint #1"
DataReader.readAndSaveData(projectName, milestoneName, dataDir)(privateToken, sparkSession)
//DataReader.getNotesPath().foreach(println)
val issuesPath = s"$dataDir/issues.json"
val milestonePath = s"$dataDir/milestone.json"
val labelsPath = s"$dataDir/labels.json"
val issuesDF = sparkSession.read.json(issuesPath).as[Issue]
val milestoneDF = sparkSession.read.json(milestonePath).as[Milestone]
val labelsDF = sparkSession.read.json(labelsPath).as[Label]
val notesDF = sparkSession.read.json(DataReader.getNotesPath(dataDir): _*).as[Note]
//issuesDF.show()
//milestoneDF.show()
//labelsDF.show()
//notesDF.show()
val labelsScoreMap = getLabelsScore(labelsDF).map(labelScore => (labelScore.id -> labelScore.score)).toMap
val issuesIdIIdMap = issuesDF.map(issue => (issue.id -> issue.iid)).collect().toMap[Long, Long]
//milestone start dans due date
val milestone = milestoneDF.filter(_.title == milestoneName).head()
val startDate = milestone.start_date
val dueDate = Date.valueOf(java.time.LocalDate.ofEpochDay(milestone.due_date.toLocalDate.toEpochDay + 1))
//Building an history of issues score evolution
val addRemoveOperationsHistory = notesDF.map(note => processAddRemoveNoteOperation(note, labelsScoreMap, issuesIdIIdMap))
.collect().flatten.toSeq.toDF("id","date","timestamp", "operation","value").as[Operation]
val opH = addRemoveOperationsHistory.collect().toSeq
val inferedAddOperations = inferNoNotedOperations(startDate, opH).toDF("id","date","timestamp", "operation","value").as[Operation]
val phs = addRemoveOperationsHistory.union(inferedAddOperations).collect().toSeq
val closedOperationsHistory = notesDF.map(note => processClosedNoteOperation(note, labelsScoreMap, issuesIdIIdMap, phs))
.collect().flatten.toSeq.toDF("id","date","timestamp", "operation","value").as[Operation]
val operationsHistory = addRemoveOperationsHistory.union(closedOperationsHistory).union(inferedAddOperations).sort(desc("id"),desc("timestamp"))
//Calculate the initial score of the Milestone
val initialScore = processInitialScore(startDate, operationsHistory, issuesDF)
//building ideal and actual graph
val idealGraphValues = buildIdealLineGraphValues(startDate, dueDate, initialScore)
idealGraphValues.foreach(println)
val actualGraphValues = buildActualLineGraphValues(startDate, operationsHistory, initialScore, idealGraphValues)
actualGraphValues.foreach(println)
val html = GraphBuilder.getHTMLGraphCode(idealGraphValues, actualGraphValues , title)
DataReader.writeFile(dataDir+"/chart.html", html)
println(s"Burndown chart saved on ${dataDir}/chart.html")
}
def inferNoNotedOperations(startDate: Date, operations: Seq[Operation]): Seq[Operation] = {
val ids = operations.map(_.id).distinct
val result = ids.map(id => {
val ops = operations.filter(_.id == id)
val filtred = ops.filter(_.operation.equals(REMOVE))
filtred.map(op => {
val addOperationForRemove = ops.filter(add => add.operation == ADD && add.timestamp.before(op.timestamp) && add.value == -1f * op.value)
if(addOperationForRemove.isEmpty){
Seq(Operation(op.id, startDate, Timestamp.valueOf(startDate.toLocalDate.atTime(00,00)),ADD, -1f * op.value))
}else
{
Seq()
}
})
}
).flatten.flatten
result
}
def buildActualLineGraphValues(startDate: Date, operationsHistory: Dataset[Operation], initialScore: Float, idealGraphValues: Seq[GraphValue])(implicit sparkSession: SparkSession): Seq[GraphValue] ={
import sparkSession.implicits._
var total = initialScore.toDouble
def dec(value: Double):Double ={
val tmp_total = total + value
total = tmp_total
tmp_total
}
//https://ci.linagora.com/linagora/lgs/openpaas/linagora.esn.chat/milestones/11
val operationsValue = operationsHistory
.filter($"date" > startDate)
.select($"date", $"value")
.groupBy($"date")
.sum("value")
.sort(asc("date"))
.collect()
val graphValues = operationsValue
.map(row => (row.getAs[Date]("date"), dec(row.getAs[Double]("sum(value)"))))
.map(value => GraphValue(value._1, isWeekend(value._1), value._2.toFloat))
.toSeq
val actualGraphDates = graphValues.map(_.date)
val now = Date.valueOf(java.time.LocalDate.now())
val weekendDays = idealGraphValues.filter(igv => igv.isWeekend == true
&& (igv.date.before(now) || igv.date == now)
&& !actualGraphDates.contains(igv.date))
val actualGraphValues = Seq(idealGraphValues.head) ++ graphValues ++ weekendDays
val sortedActualGraphValues = actualGraphValues.sortWith((gv1,gv2)=> gv1.date.before(gv2.date))
var dateValues = sortedActualGraphValues.map(gv => (gv.date -> gv.value)).toMap
sortedActualGraphValues.map(_.date).foreach(println)
sortedActualGraphValues.map(gv => if(gv.isWeekend){
val value = dateValues.get(getFirstActiveDateBefore(sortedActualGraphValues.map(_.date), gv.date))
if (value == None) println(s"Date before ${gv.date} is "+ getDateBefore(gv.date))
dateValues = dateValues.updated(gv.date, value.get)
GraphValue(gv.date, gv.isWeekend, value.get)
}else{
gv
})
}
private def getDateBefore(date: Date): Date = {
Date.valueOf(java.time.LocalDate.ofEpochDay(date.toLocalDate.toEpochDay - 1L))
}
private def getFirstActiveDateBefore(dates: Seq[Date], date: Date): Date = {
var dateBefore = java.sql.Date.valueOf(java.time.LocalDate.ofEpochDay(date.toLocalDate.toEpochDay - 1L))
var inc = 1L
while(!dates.contains(dateBefore) && !dateBefore.equals(dates.head)){
inc = inc + 1L
dateBefore = java.sql.Date.valueOf(java.time.LocalDate.ofEpochDay(date.toLocalDate.toEpochDay - inc))
}
dateBefore
}
private def processInitialScore(startDate: Date, operationsHistory: Dataset[Operation], issuesDF: Dataset[Issue])(implicit sparkSession: SparkSession): Float = {
import sparkSession.implicits._
//val currentScore = processCurrentScore(issuesDF)
val scoreWithoutHistory = processScoreWithoutNotes(issuesDF, operationsHistory)
val operationsValue = operationsHistory
.select($"date", $"value")
.groupBy($"date")
.sum("value")
.sort(desc("date"))
operationsValue.show()
val initialScore = operationsValue
.filter($"date" <= startDate)
.select(sum("sum(value)").alias("estimation"))
.head().getAs[Double]("estimation").toFloat + scoreWithoutHistory
/* val scoreEvolution = operationsValue
.filter($"date" > startDate)
.select(sum("sum(value)").alias("estimation"))
.head().getAs[Double]("estimation").toFloat*/
//println(s"Current score $currentScore")
println(s"Score without history $scoreWithoutHistory")