Commit 8eb09276 authored by Edwin Carlinet's avatar Edwin Carlinet
Browse files

Use Lab transformation for detection and automatic method

selection. Fix ffmpeg decoding loop.

           *  apps/smartdoc/analyse_xml.py: New
           *  apps/smartdoc/main2.cpp: Use Lab colorspace and fix
              ffmpeg loop.
           *  apps/smartdoc/smartdoc.cpp: Fix size parameter.
parent 876094bf
#! /usr/bin/python
import sys
import xml.etree.ElementTree as ET
import numpy as np
import matplotlib.pyplot as plt
import matplotlib as mpl
import datetime
from numpy.linalg import norm
from scipy.interpolate import Rbf, InterpolatedUnivariateSpline
from scipy.ndimage.filters import convolve
if len(sys.argv) < 3:
print "Usage:", sys.argv[0], "fichier.xml out.png [out.xml]"
tree = ET.parse(sys.argv[1])
frames = tree.getroot().findall("*/frame")
def getPoint(point):
x = float(point.get("x"))
y = float(point.get("y"))
return (x,y)
def getPoints(frame):
tl = getPoint(frame.find("./point[@name='tl']"))
tr = getPoint(frame.find("./point[@name='tr']"))
bl = getPoint(frame.find("./point[@name='bl']"))
br = getPoint(frame.find("./point[@name='br']"))
return [tl, tr, bl, br]
def getDist(frame):
if frame.get("rejected") == "True":
return [0, 0, 0, 0]
def export_xml(y, filename):
f = open(filename, "w")
f.write("<?xml version='1.0' encoding='utf-8'?>'\n")
f.write('<seg_result version="0.2" generated="%s">\n' % datetime.datetime.now().isoformat())
f.write('<software_used name="MILENA" version="2.0"/>\n')
idx = filename.index("background")
f.write('<source_sample_file>%s.avi</source_sample_file>\n' % filename[idx:])
f.write(' <segmentation_results>\n')
for i in xrange(len(y)):
f.write(' <frame index="%i" rejected="false">\n' % (i+1))
f.write(' <point name="tl" x="%i" y="%i" />\n' % (y[i,0,0], y[i,0,1]))
f.write(' <point name="tr" x="%i" y="%i" />\n' % (y[i,1,0], y[i,1,1]))
f.write(' <point name="bl" x="%i" y="%i" />\n' % (y[i,2,0], y[i,2,1]))
f.write(' <point name="br" x="%i" y="%i" />\n' % (y[i,3,0], y[i,3,1]))
f.write(' </frame>')
f.write(' </segmentation_results>\n')
f.write('</seg_result>\n')
pass
points = [ getPoints(f) for f in frames ]
points = np.array(points)
x = np.arange(len(points))
a = norm((points[:,1] - points[:,0]), axis = 1)
b = norm((points[:,3] - points[:,1]), axis = 1)
c = norm((points[:,3] - points[:,0]), axis = 1)
d = norm((points[:,3] - points[:,2]), axis = 1)
e = norm((points[:,2] - points[:,0]), axis = 1)
s = (a+b+c)/2.0
a1 = np.sqrt(s*(s-a)*(s-b)*(s-c))
s = (c+d+e)/2.0
a2 = np.sqrt(s*(s-c)*(s-d)*(s-e))
area = a1 + a2
perim = a + b + d + e
# Detect bad points
# last = 2
# firsterr = True
# area[last] = area[:last+1].mean()
# for i in xrange(last+1, len(points)):
# maxdist = np.abs(points[i] - points[last]).max()
# if maxdist > (80 + 5 * (i-last)):
# #maxdist = abs(area[i] - area[last]) / area[last]
# #if maxdist > 0.20: #if 10% d'ecart
# if firsterr:
# print sys.argv[1]
# firsterr = False
# print "Frame #%i dismissed (%f)" % (i, maxdist)
# mask[i] = False
# # if (i - last > 5) and (np.std(area[last-5:last]) / area.mean()) < 0.05:
# # last = i
# # mask[last-5:last] = True
# # else:
# # mask[i] = False
# else:
# last = i
# mask[-1] = True
# interpolate missing values
y = np.empty(points.shape)
mask = np.ones( (len(points)), dtype=bool)
firsterr = True
for k in xrange(4):
mymask = np.ones( (len(points)), dtype=bool)
last = 2
for i in xrange(last+1, len(points)):
maxdist = np.abs(points[i,k] - points[last,k]).max()
if maxdist > (100 + 5 * (i-last)):
if firsterr:
print sys.argv[1]
firsterr = False
print "Frame #%i dismissed (%f)" % (i, maxdist)
mymask[i] = False
else:
last = i
f = InterpolatedUnivariateSpline(x[mymask], points[mymask,k,0], k=1)
y[:,k,0] = f(x)
f = InterpolatedUnivariateSpline(x[mymask], points[mymask,k,1], k=1)
y[:,k,1] = f(x)
mask = np.logical_and(mask,mymask)
avg = np.empty(points.shape)
for i in xrange(4):
avg[:,i,0] = convolve(points[:,i,0], [1,1,1,1,1]) / 5.0
avg[:,i,1] = convolve(points[:,i,1], [1,1,1,1,1]) / 5.0
plt.clf()
for i in range(4):
plt.subplot(5,2,i*2+1)
plt.plot(x, points[:,i,0], '+')
plt.plot(x, y[:,i,0], 'r')
plt.plot(x, avg[:,i,0] + 50, 'g')
plt.plot(x, avg[:,i,0] - 50, 'b')
plt.subplot(5,2,i*2+2)
plt.plot(x, points[:,i,1], '+')
plt.plot(x, y[:,i,1], 'r')
plt.plot(x, avg[:,i,1] + 50, 'g')
plt.plot(x, avg[:,i,1] - 50, 'b')
plt.subplot("529")
plt.plot(x, area)
plt.subplot(5,2,10)
plt.fill(x, np.logical_not(mask))
# plt.scatter(points[:,0,0], points[:,0,1], c = np.arange(len(points)) / float(len(points)),
# cmap=mpl.cm.Blues)
# plt.scatter(points[:,1,0], points[:,1,1], c = np.arange(len(points)) / float(len(points)),
# cmap=mpl.cm.Reds)
# plt.scatter(points[:,2,0], points[:,2,1], c = np.arange(len(points)) / float(len(points)),
# cmap=mpl.cm.spring)
# plt.scatter(points[:,3,0], points[:,3,1], c = np.arange(len(points)) / float(len(points)),
# cmap=mpl.cm.winter)
#plt.show()
plt.savefig(sys.argv[2], dpi=300)
if len(sys.argv) > 3:
export_xml(y, sys.argv[3])
#include <mln/core/image/image2d.hpp>
#include <mln/core/image/morphers/casted_image.hpp>
#include <mln/core/colors.hpp>
#include <mln/core/trace.hpp>
#include <mln/core/algorithm/transform.hpp>
#include <mln/io/imsave.hpp>
#include <mln/colors/lab.hpp>
#include <mln/morpho/tos/ctos.hpp>
#include <mln/morpho/component_tree/compute_depth.hpp>
#include <mln/morpho/component_tree/reconstruction.hpp>
#include <mln/morpho/structural/opening.hpp>
#include <mln/morpho/structural/closing.hpp>
#include <tbb/pipeline.h>
#include <boost/format.hpp>
#include <mutex>
#include <apps/g2/compute_ctos.hpp>
#include <apps/tos/addborder.hpp>
#include <mln/io/imsave.hpp>
#include "video_tools.hh"
#include "smartdoc.hpp"
......@@ -19,12 +28,18 @@
const bool ENABLE_CACHING = true;
bool VIDEO_OUTPUT = false; // non-const modifiable by the cmdline
bool USE_LAB = true;
enum e_method { ON_LAB_L = 0, ON_LAB_B = 1, BACKUP_METHOD = 2, AUTO = 3};
e_method APP_METHOD = ON_LAB_L;
/****************************************************/
/** PROCESSORS ***/
/****************************************************/
std::mutex my_mutex;
/*
We have a pipeline with 3 filters:
......@@ -95,33 +110,60 @@ public:
avcodec_decode_video2(Ctx_Dec, pFrame_inYUV, &frameFinished, &packet_dec);
if (frameFinished)
{
/* convert from YUV to RGB24 to decode video */
sws_scale(convert_ctx_yuv2rgb,
(const uint8_t * const*) pFrame_inYUV->data,
pFrame_inYUV->linesize,
0,
Ctx_Dec->height,
pFrame_inRGB->data,
pFrame_inRGB->linesize);
/* copy to mln image2d */
mln::image2d<mln::rgb8>* out = new mln::image2d<mln::rgb8> ();
size_t strides[2] = {(size_t)pFrame_inRGB->linesize[0], sizeof(mln::rgb8)};
mln::box2d domain{{0,0},{(short)pFrame_inRGB->height, (short)pFrame_inRGB->width}};
*out = mln::image2d<mln::rgb8>::from_buffer(pFrame_inRGB->data[0],
domain, strides, true);
*out = this->getCurrentFrame();
++m_nbframe;
return out;
}
}
}
packet_dec.data = NULL;
packet_dec.size = 0;
packet_dec.stream_index = m_videoStream;
while (avcodec_decode_video2(Ctx_Dec, pFrame_inYUV,
&frameFinished, &packet_dec) >= 0)
{
if (frameFinished)
{
mln::image2d<mln::rgb8>* out = new mln::image2d<mln::rgb8> ();
*out = this->getCurrentFrame();
++m_nbframe;
return out;
}
else
{
break;
}
}
fc.stop(); // NO MORE FRAME
return NULL;
}
mln::image2d<mln::rgb8>
getCurrentFrame() const
{
/* convert from YUV to RGB24 to decode video */
sws_scale(convert_ctx_yuv2rgb,
(const uint8_t * const*) pFrame_inYUV->data,
pFrame_inYUV->linesize,
0,
Ctx_Dec->height,
pFrame_inRGB->data,
pFrame_inRGB->linesize);
/* copy to mln image2d */
size_t strides[2] = {(size_t)pFrame_inRGB->linesize[0], sizeof(mln::rgb8)};
mln::box2d domain{{0,0},{(short)pFrame_inRGB->height,
(short)pFrame_inRGB->width}};
return mln::image2d<mln::rgb8>::from_buffer(pFrame_inRGB->data[0],
domain, strides, true);
}
AVCodecContext*
get_context()
{
......@@ -162,6 +204,8 @@ public:
middle_filter_result*
operator() (mln::image2d<mln::rgb8>* input) const
{
using namespace mln;
// Process
middle_filter_result* mdr = new middle_filter_result;
......@@ -173,7 +217,140 @@ public:
// ptr_img_out = &(mdr->ima);
// }
auto tree = compute_ctos(*input, &(mdr->depth));
typedef morpho::component_tree< unsigned, image2d<unsigned> > tree_t;
tree_t tree;
if (not USE_LAB)
{
tree = compute_ctos(*input, &(mdr->depth));
}
else
{
//std::array<process_result_t, NQUAD> resL, resB;
//image2d< lab<float> > f = transform(*input, [](rgb8 x) { return rgb2lab(x); });
image2d<uint8> f;
property_map<tree_t, unsigned> vmap;
my_mutex.lock();
if (APP_METHOD == AUTO)
{
std::array<process_result_t, NQUAD> resL, resB;
// 1st try on LAB_B
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return (v[2] + 110) * 256 / 220;
});
f = addborder(f);
tree = morpho::cToS(f, c4);
resize(mdr->depth, tree._get_data()->m_pmap);
vmap = morpho::compute_depth(tree);
morpho::reconstruction(tree, vmap, mdr->depth);
resB = process(tree, mdr->depth);
}
if (resB[0].energy > 0.78)
{
std::cout << "==> Using the B component." << std::endl;
APP_METHOD = ON_LAB_B;
mdr->res = resB;
my_mutex.unlock();
delete input;
return mdr;
}
// 2nd try on LAB_L
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return v[0] * 256 / 100;
});
f = addborder(f);
tree = morpho::cToS(f, c4);
vmap = morpho::compute_depth(tree);
morpho::reconstruction(tree, vmap, mdr->depth);
resL = process(tree, mdr->depth);
}
if (resB[0].energy > 0.72 and resL[0].energy < 0.8)
{
std::cout << "==> Using the B component." << std::endl;
APP_METHOD = ON_LAB_B;
mdr->res = resB;
my_mutex.unlock();
delete input;
return mdr;
}
else if (resL[0].energy > 0.8)
{
std::cout << "==> Using the L component." << std::endl;
APP_METHOD = ON_LAB_L;
mdr->res = resL;
my_mutex.unlock();
delete input;
return mdr;
}
// 3. Backup
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return (v[2] + 110) * 256 / 220;
});
rect2d r = make_rectangle2d(101, 1);
f = morpho::structural::opening(f, r);
f = morpho::structural::closing(f, r);
f = addborder(f);
tree = morpho::cToS(f, c4);
vmap = morpho::compute_depth(tree);
morpho::reconstruction(tree, vmap, mdr->depth);
mdr->res = process(tree, mdr->depth);
}
{
std::cout << "==> Using the BACKUP method." << std::endl;
APP_METHOD = BACKUP_METHOD;
my_mutex.unlock();
delete input;
return mdr;
}
}
// We now which method to run OK.
my_mutex.unlock();
if (APP_METHOD == ON_LAB_B)
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return (v[2] + 110) * 256 / 220;
});
}
else if (APP_METHOD == ON_LAB_L)
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return v[0] * 256 / 100;
});
}
else if (APP_METHOD == BACKUP_METHOD)
{
f = transform(*input, [](rgb8 x) -> uint8 {
lab<float> v = rgb2lab(x);
return (v[2] + 110) * 256 / 220;
});
rect2d r = make_rectangle2d(101, 1);
f = morpho::structural::opening(f, r);
f = morpho::structural::closing(f, r);
}
f = addborder(f);
tree = morpho::cToS(f, c4);
resize(mdr->depth, tree._get_data()->m_pmap);
vmap = morpho::compute_depth(tree);
morpho::reconstruction(tree, vmap, mdr->depth);
}
mdr->res = process(tree, mdr->depth);
delete input;
......@@ -340,15 +517,20 @@ private:
int main(int argc, char** argv)
{
if (argc < 3) {
std::cerr << "Usage: " << argv[0] << " input.mpg output.xml [number of threads=10] [output.mpg]\n";
std::cerr << "Usage: " << argv[0] << " input.mpg output.xml [method = 0] [number of threads=10] [output.mpg]\n"
"method: (0) LAB_L, (1): LAB_B, (2): BACKUP\n";
std::exit(1);
}
const char* input_path = argv[1];
const char* output_path = argv[2];
int nthread = (argc > 3) ? std::atoi(argv[3]) : 10;
const char* output_video_path = (argc > 4) ? argv[4] : NULL;
VIDEO_OUTPUT = argc > 4;
if (argc > 3) {
APP_METHOD = (e_method) std::atoi(argv[3]);
}
int nthread = (argc > 4) ? std::atoi(argv[4]) : 10;
const char* output_video_path = (argc > 5) ? argv[5] : NULL;
VIDEO_OUTPUT = argc > 5;
using namespace mln;
......
......@@ -11,7 +11,7 @@
#include <apps/tos/croutines.hpp>
static constexpr int EXTINCTION_THRESHOLD = 0.05;
static constexpr int MINIMUM_ABSOLUTE_SIZE = 400 * 400 * 4; // 400*400*4
static constexpr int MINIMUM_ABSOLUTE_SIZE = 300000; // 500 * 500; // 400*400*4
namespace mln
{
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment