summaryrefslogtreecommitdiffstats
path: root/third_party/aom/test/visual_metrics.py
diff options
context:
space:
mode:
Diffstat (limited to 'third_party/aom/test/visual_metrics.py')
-rwxr-xr-xthird_party/aom/test/visual_metrics.py466
1 files changed, 466 insertions, 0 deletions
diff --git a/third_party/aom/test/visual_metrics.py b/third_party/aom/test/visual_metrics.py
new file mode 100755
index 000000000..9055feb33
--- /dev/null
+++ b/third_party/aom/test/visual_metrics.py
@@ -0,0 +1,466 @@
+#!/usr/bin/python
+#
+# Copyright (c) 2016, Alliance for Open Media. All rights reserved
+#
+# This source code is subject to the terms of the BSD 2 Clause License and
+# the Alliance for Open Media Patent License 1.0. If the BSD 2 Clause License
+# was not distributed with this source code in the LICENSE file, you can
+# obtain it at www.aomedia.org/license/software. If the Alliance for Open
+# Media Patent License 1.0 was not distributed with this source code in the
+# PATENTS file, you can obtain it at www.aomedia.org/license/patent.
+#
+
+"""Converts video encoding result data from text files to visualization
+data source."""
+
+__author__ = "jzern@google.com (James Zern),"
+__author__ += "jimbankoski@google.com (Jim Bankoski)"
+
+import fnmatch
+import numpy as np
+import scipy as sp
+import scipy.interpolate
+import os
+import re
+import string
+import sys
+import math
+import warnings
+
+import gviz_api
+
+from os.path import basename
+from os.path import splitext
+
+warnings.simplefilter('ignore', np.RankWarning)
+warnings.simplefilter('ignore', RuntimeWarning)
+
+def bdsnr2(metric_set1, metric_set2):
+ """
+ BJONTEGAARD Bjontegaard metric calculation adapted
+ Bjontegaard's snr metric allows to compute the average % saving in decibels
+ between two rate-distortion curves [1]. This is an adaptation of that
+ method that fixes inconsistencies when the curve fit operation goes awry
+ by replacing the curve fit function with a Piecewise Cubic Hermite
+ Interpolating Polynomial and then integrating that by evaluating that
+ function at small intervals using the trapezoid method to calculate
+ the integral.
+
+ metric_set1 - list of tuples ( bitrate, metric ) for first graph
+ metric_set2 - list of tuples ( bitrate, metric ) for second graph
+ """
+
+ if not metric_set1 or not metric_set2:
+ return 0.0
+
+ try:
+
+ # pchip_interlopate requires keys sorted by x axis. x-axis will
+ # be our metric not the bitrate so sort by metric.
+ metric_set1.sort()
+ metric_set2.sort()
+
+ # Pull the log of the rate and clamped psnr from metric_sets.
+ log_rate1 = [math.log(x[0]) for x in metric_set1]
+ metric1 = [100.0 if x[1] == float('inf') else x[1] for x in metric_set1]
+ log_rate2 = [math.log(x[0]) for x in metric_set2]
+ metric2 = [100.0 if x[1] == float('inf') else x[1] for x in metric_set2]
+
+ # Integration interval. This metric only works on the area that's
+ # overlapping. Extrapolation of these things is sketchy so we avoid.
+ min_int = max([min(log_rate1), min(log_rate2)])
+ max_int = min([max(log_rate1), max(log_rate2)])
+
+ # No overlap means no sensible metric possible.
+ if max_int <= min_int:
+ return 0.0
+
+ # Use Piecewise Cubic Hermite Interpolating Polynomial interpolation to
+ # create 100 new samples points separated by interval.
+ lin = np.linspace(min_int, max_int, num=100, retstep=True)
+ interval = lin[1]
+ samples = lin[0]
+ v1 = scipy.interpolate.pchip_interpolate(log_rate1, metric1, samples)
+ v2 = scipy.interpolate.pchip_interpolate(log_rate2, metric2, samples)
+
+ # Calculate the integral using the trapezoid method on the samples.
+ int_v1 = np.trapz(v1, dx=interval)
+ int_v2 = np.trapz(v2, dx=interval)
+
+ # Calculate the average improvement.
+ avg_exp_diff = (int_v2 - int_v1) / (max_int - min_int)
+
+ except (TypeError, ZeroDivisionError, ValueError, np.RankWarning) as e:
+ return 0
+
+ return avg_exp_diff
+
+def bdrate2(metric_set1, metric_set2):
+ """
+ BJONTEGAARD Bjontegaard metric calculation adapted
+ Bjontegaard's metric allows to compute the average % saving in bitrate
+ between two rate-distortion curves [1]. This is an adaptation of that
+ method that fixes inconsistencies when the curve fit operation goes awry
+ by replacing the curve fit function with a Piecewise Cubic Hermite
+ Interpolating Polynomial and then integrating that by evaluating that
+ function at small intervals using the trapezoid method to calculate
+ the integral.
+
+ metric_set1 - list of tuples ( bitrate, metric ) for first graph
+ metric_set2 - list of tuples ( bitrate, metric ) for second graph
+ """
+
+ if not metric_set1 or not metric_set2:
+ return 0.0
+
+ try:
+
+ # pchip_interlopate requires keys sorted by x axis. x-axis will
+ # be our metric not the bitrate so sort by metric.
+ metric_set1.sort(key=lambda tup: tup[1])
+ metric_set2.sort(key=lambda tup: tup[1])
+
+ # Pull the log of the rate and clamped psnr from metric_sets.
+ log_rate1 = [math.log(x[0]) for x in metric_set1]
+ metric1 = [100.0 if x[1] == float('inf') else x[1] for x in metric_set1]
+ log_rate2 = [math.log(x[0]) for x in metric_set2]
+ metric2 = [100.0 if x[1] == float('inf') else x[1] for x in metric_set2]
+
+ # Integration interval. This metric only works on the area that's
+ # overlapping. Extrapolation of these things is sketchy so we avoid.
+ min_int = max([min(metric1), min(metric2)])
+ max_int = min([max(metric1), max(metric2)])
+
+ # No overlap means no sensible metric possible.
+ if max_int <= min_int:
+ return 0.0
+
+ # Use Piecewise Cubic Hermite Interpolating Polynomial interpolation to
+ # create 100 new samples points separated by interval.
+ lin = np.linspace(min_int, max_int, num=100, retstep=True)
+ interval = lin[1]
+ samples = lin[0]
+ v1 = scipy.interpolate.pchip_interpolate(metric1, log_rate1, samples)
+ v2 = scipy.interpolate.pchip_interpolate(metric2, log_rate2, samples)
+
+ # Calculate the integral using the trapezoid method on the samples.
+ int_v1 = np.trapz(v1, dx=interval)
+ int_v2 = np.trapz(v2, dx=interval)
+
+ # Calculate the average improvement.
+ avg_exp_diff = (int_v2 - int_v1) / (max_int - min_int)
+
+ except (TypeError, ZeroDivisionError, ValueError, np.RankWarning) as e:
+ return 0
+
+ # Convert to a percentage.
+ avg_diff = (math.exp(avg_exp_diff) - 1) * 100
+
+ return avg_diff
+
+
+
+def FillForm(string_for_substitution, dictionary_of_vars):
+ """
+ This function substitutes all matches of the command string //%% ... %%//
+ with the variable represented by ... .
+ """
+ return_string = string_for_substitution
+ for i in re.findall("//%%(.*)%%//", string_for_substitution):
+ return_string = re.sub("//%%" + i + "%%//", dictionary_of_vars[i],
+ return_string)
+ return return_string
+
+
+def HasMetrics(line):
+ """
+ The metrics files produced by aomenc are started with a B for headers.
+ """
+ # If the first char of the first word on the line is a digit
+ if len(line) == 0:
+ return False
+ if len(line.split()) == 0:
+ return False
+ if line.split()[0][0:1].isdigit():
+ return True
+ return False
+
+def GetMetrics(file_name):
+ metric_file = open(file_name, "r")
+ return metric_file.readline().split();
+
+def ParseMetricFile(file_name, metric_column):
+ metric_set1 = set([])
+ metric_file = open(file_name, "r")
+ for line in metric_file:
+ metrics = string.split(line)
+ if HasMetrics(line):
+ if metric_column < len(metrics):
+ try:
+ tuple = float(metrics[0]), float(metrics[metric_column])
+ except:
+ tuple = float(metrics[0]), 0
+ else:
+ tuple = float(metrics[0]), 0
+ metric_set1.add(tuple)
+ metric_set1_sorted = sorted(metric_set1)
+ return metric_set1_sorted
+
+
+def FileBetter(file_name_1, file_name_2, metric_column, method):
+ """
+ Compares two data files and determines which is better and by how
+ much. Also produces a histogram of how much better, by PSNR.
+ metric_column is the metric.
+ """
+ # Store and parse our two files into lists of unique tuples.
+
+ # Read the two files, parsing out lines starting with bitrate.
+ metric_set1_sorted = ParseMetricFile(file_name_1, metric_column)
+ metric_set2_sorted = ParseMetricFile(file_name_2, metric_column)
+
+
+ def GraphBetter(metric_set1_sorted, metric_set2_sorted, base_is_set_2):
+ """
+ Search through the sorted metric file for metrics on either side of
+ the metric from file 1. Since both lists are sorted we really
+ should not have to search through the entire range, but these
+ are small files."""
+ total_bitrate_difference_ratio = 0.0
+ count = 0
+ for bitrate, metric in metric_set1_sorted:
+ if bitrate == 0:
+ continue
+ for i in range(len(metric_set2_sorted) - 1):
+ s2_bitrate_0, s2_metric_0 = metric_set2_sorted[i]
+ s2_bitrate_1, s2_metric_1 = metric_set2_sorted[i + 1]
+ # We have a point on either side of our metric range.
+ if metric > s2_metric_0 and metric <= s2_metric_1:
+
+ # Calculate a slope.
+ if s2_metric_1 - s2_metric_0 != 0:
+ metric_slope = ((s2_bitrate_1 - s2_bitrate_0) /
+ (s2_metric_1 - s2_metric_0))
+ else:
+ metric_slope = 0
+
+ estimated_s2_bitrate = (s2_bitrate_0 + (metric - s2_metric_0) *
+ metric_slope)
+
+ if estimated_s2_bitrate == 0:
+ continue
+ # Calculate percentage difference as given by base.
+ if base_is_set_2 == 0:
+ bitrate_difference_ratio = ((bitrate - estimated_s2_bitrate) /
+ bitrate)
+ else:
+ bitrate_difference_ratio = ((bitrate - estimated_s2_bitrate) /
+ estimated_s2_bitrate)
+
+ total_bitrate_difference_ratio += bitrate_difference_ratio
+ count += 1
+ break
+
+ # Calculate the average improvement between graphs.
+ if count != 0:
+ avg = total_bitrate_difference_ratio / count
+
+ else:
+ avg = 0.0
+
+ return avg
+
+ # Be fair to both graphs by testing all the points in each.
+ if method == 'avg':
+ avg_improvement = 50 * (
+ GraphBetter(metric_set1_sorted, metric_set2_sorted, 1) -
+ GraphBetter(metric_set2_sorted, metric_set1_sorted, 0))
+ elif method == 'dsnr':
+ avg_improvement = bdsnr2(metric_set1_sorted, metric_set2_sorted)
+ else:
+ avg_improvement = bdrate2(metric_set2_sorted, metric_set1_sorted)
+
+ return avg_improvement
+
+
+def HandleFiles(variables):
+ """
+ This script creates html for displaying metric data produced from data
+ in a video stats file, as created by the AOM project when enable_psnr
+ is turned on:
+
+ Usage: visual_metrics.py template.html pattern base_dir sub_dir [ sub_dir2 ..]
+
+ The script parses each metrics file [see below] that matches the
+ statfile_pattern in the baseline directory and looks for the file that
+ matches that same file in each of the sub_dirs, and compares the resultant
+ metrics bitrate, avg psnr, glb psnr, and ssim. "
+
+ It provides a table in which each row is a file in the line directory,
+ and a column for each subdir, with the cells representing how that clip
+ compares to baseline for that subdir. A graph is given for each which
+ compares filesize to that metric. If you click on a point in the graph it
+ zooms in on that point.
+
+ a SAMPLE metrics file:
+
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 25.911 38.242 38.104 38.258 38.121 75.790 14103
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 49.982 41.264 41.129 41.255 41.122 83.993 19817
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 74.967 42.911 42.767 42.899 42.756 87.928 17332
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 100.012 43.983 43.838 43.881 43.738 89.695 25389
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 149.980 45.338 45.203 45.184 45.043 91.591 25438
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 199.852 46.225 46.123 46.113 45.999 92.679 28302
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 249.922 46.864 46.773 46.777 46.673 93.334 27244
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 299.998 47.366 47.281 47.317 47.220 93.844 27137
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 349.769 47.746 47.677 47.722 47.648 94.178 32226
+ Bitrate AVGPsnr GLBPsnr AVPsnrP GLPsnrP VPXSSIM Time(us)
+ 399.773 48.032 47.971 48.013 47.946 94.362 36203
+
+ sample use:
+ visual_metrics.py template.html "*stt" aom aom_b aom_c > metrics.html
+ """
+
+ # The template file is the html file into which we will write the
+ # data from the stats file, formatted correctly for the gviz_api.
+ template_file = open(variables[1], "r")
+ page_template = template_file.read()
+ template_file.close()
+
+ # This is the path match pattern for finding stats files amongst
+ # all the other files it could be. eg: *.stt
+ file_pattern = variables[2]
+
+ # This is the directory with files that we will use to do the comparison
+ # against.
+ baseline_dir = variables[3]
+ snrs = ''
+ filestable = {}
+
+ filestable['dsnr'] = ''
+ filestable['drate'] = ''
+ filestable['avg'] = ''
+
+ # Dirs is directories after the baseline to compare to the base.
+ dirs = variables[4:len(variables)]
+
+ # Find the metric files in the baseline directory.
+ dir_list = sorted(fnmatch.filter(os.listdir(baseline_dir), file_pattern))
+
+ metrics = GetMetrics(baseline_dir + "/" + dir_list[0])
+
+ metrics_js = 'metrics = ["' + '", "'.join(metrics) + '"];'
+
+ for column in range(1, len(metrics)):
+
+ for metric in ['avg','dsnr','drate']:
+ description = {"file": ("string", "File")}
+
+ # Go through each directory and add a column header to our description.
+ countoverall = {}
+ sumoverall = {}
+
+ for directory in dirs:
+ description[directory] = ("number", directory)
+ countoverall[directory] = 0
+ sumoverall[directory] = 0
+
+ # Data holds the data for the visualization, name given comes from
+ # gviz_api sample code.
+ data = []
+ for filename in dir_list:
+ row = {'file': splitext(basename(filename))[0] }
+ baseline_file_name = baseline_dir + "/" + filename
+
+ # Read the metric file from each of the directories in our list.
+ for directory in dirs:
+ metric_file_name = directory + "/" + filename
+
+ # If there is a metric file in the current directory, open it
+ # and calculate its overall difference between it and the baseline
+ # directory's metric file.
+ if os.path.isfile(metric_file_name):
+ overall = FileBetter(baseline_file_name, metric_file_name,
+ column, metric)
+ row[directory] = overall
+
+ sumoverall[directory] += overall
+ countoverall[directory] += 1
+
+ data.append(row)
+
+ # Add the overall numbers.
+ row = {"file": "OVERALL" }
+ for directory in dirs:
+ row[directory] = sumoverall[directory] / countoverall[directory]
+ data.append(row)
+
+ # write the tables out
+ data_table = gviz_api.DataTable(description)
+ data_table.LoadData(data)
+
+ filestable[metric] = ( filestable[metric] + "filestable_" + metric +
+ "[" + str(column) + "]=" +
+ data_table.ToJSon(columns_order=["file"]+dirs) + "\n" )
+
+ filestable_avg = filestable['avg']
+ filestable_dpsnr = filestable['dsnr']
+ filestable_drate = filestable['drate']
+
+ # Now we collect all the data for all the graphs. First the column
+ # headers which will be Datarate and then each directory.
+ columns = ("datarate",baseline_dir)
+ description = {"datarate":("number", "Datarate")}
+ for directory in dirs:
+ description[directory] = ("number", directory)
+
+ description[baseline_dir] = ("number", baseline_dir)
+
+ snrs = snrs + "snrs[" + str(column) + "] = ["
+
+ # Now collect the data for the graphs, file by file.
+ for filename in dir_list:
+
+ data = []
+
+ # Collect the file in each directory and store all of its metrics
+ # in the associated gviz metrics table.
+ all_dirs = dirs + [baseline_dir]
+ for directory in all_dirs:
+
+ metric_file_name = directory + "/" + filename
+ if not os.path.isfile(metric_file_name):
+ continue
+
+ # Read and parse the metrics file storing it to the data we'll
+ # use for the gviz_api.Datatable.
+ metrics = ParseMetricFile(metric_file_name, column)
+ for bitrate, metric in metrics:
+ data.append({"datarate": bitrate, directory: metric})
+
+ data_table = gviz_api.DataTable(description)
+ data_table.LoadData(data)
+ snrs = snrs + "'" + data_table.ToJSon(
+ columns_order=tuple(["datarate",baseline_dir]+dirs)) + "',"
+
+ snrs = snrs + "]\n"
+
+ formatters = ""
+ for i in range(len(dirs)):
+ formatters = "%s formatter.format(better, %d);" % (formatters, i+1)
+
+ print FillForm(page_template, vars())
+ return
+
+if len(sys.argv) < 3:
+ print HandleFiles.__doc__
+else:
+ HandleFiles(sys.argv)