Commit 9d2fe866 authored by Edwin Carlinet's avatar Edwin Carlinet
Browse files

Add dependance to the previous frame result when detecting the document.

        *  apps/smartdoc/main2.cpp: Use previous frame results to
           improve detection.
        *  apps/smartdoc/smartdoc.cpp,
        *  apps/smartdoc/smartdoc.hpp: Add superimposition process as
           a standlaone function draw_quad_superimpose.
        *  apps/smartdoc/main.cpp,
        *  apps/smartdoc/smartdoc-cli.cpp: Use new interface.
parent 18d559f6
......@@ -152,10 +152,13 @@ int main(int argc, char** argv)
// Process
SmartDoc_t res;
tree = compute_ctos(img_in, &depth);
std::tie(res.good, res.quad) = process(tree, depth, ptr_img_out);
auto quads = process(tree, depth);
res = {true, quads[0].points};
// Output the video
if (VIDEO_OUTPUT) {
draw_quad_superimpose(quads[0].points, depth, *ptr_img_out);
encode_video(skipped, &pFormatCtx_Enc, output_video_path,
&pFrame_outRGB, &pFrame_outYUV, nbframe,
&Ctx_Enc, packet_enc, &convert_ctx_rgb2out);
......
......@@ -7,6 +7,7 @@
#include <boost/format.hpp>
#include <apps/g2/compute_ctos.hpp>
#include <mln/io/imsave.hpp>
#include "video_tools.hh"
#include "smartdoc.hpp"
#include "export.hpp"
......@@ -94,8 +95,6 @@ public:
avcodec_decode_video2(Ctx_Dec, pFrame_inYUV, &frameFinished, &packet_dec);
if (frameFinished)
{
std::cout << "Processing #" << m_nbframe << std::endl;
/* convert from YUV to RGB24 to decode video */
sws_scale(convert_ctx_yuv2rgb,
(const uint8_t * const*) pFrame_inYUV->data,
......@@ -148,10 +147,11 @@ private:
};
struct middle_filter_result
{
mln::image2d<mln::rgb8> ima;
SmartDoc_t res;
mln::image2d<mln::uint16> depth;
std::array<process_result_t, 3> res;
};
......@@ -162,22 +162,19 @@ public:
middle_filter_result*
operator() (mln::image2d<mln::rgb8>* input) const
{
mln::image2d<mln::uint16> depth;
mln::image2d<mln::rgb8>* ptr_img_out = NULL;
// Process
middle_filter_result* mdr = new middle_filter_result;
if (VIDEO_OUTPUT) {
mln::box2d d;
d.pmin = {0,0};
d.pmax = (input->domain().pmax + 2) * 2 - 1;
mdr->ima.resize(d);
ptr_img_out = &(mdr->ima);
}
// if (VIDEO_OUTPUT) {
// mln::box2d d;
// d.pmin = {0,0};
// d.pmax = (input->domain().pmax + 2) * 2 - 1;
// mdr->ima.resize(d);
// ptr_img_out = &(mdr->ima);
// }
auto tree = compute_ctos(*input, &depth);
std::tie(mdr->res.good, mdr->res.quad) = process(tree, depth, ptr_img_out);
auto tree = compute_ctos(*input, &(mdr->depth));
mdr->res = process(tree, mdr->depth);
delete input;
return mdr;
......@@ -201,8 +198,8 @@ public:
&pFrame_outYUV, NULL, &Ctx_Enc);
// Because the output image is in K0 + border
int w = (Ctx_Dec->width + 2) * 2 - 1;
int h = (Ctx_Dec->height + 2) * 2 - 1;
short w = (Ctx_Dec->width + 2) * 2 - 1;
short h = (Ctx_Dec->height + 2) * 2 - 1;
pFrame_outRGB = avcodec_alloc_frame();
av_image_alloc(pFrame_outRGB->data,
......@@ -219,6 +216,8 @@ public:
Ctx_Enc->height,
Ctx_Enc->pix_fmt,
SWS_BILINEAR, 0, 0, 0);
m_output.resize(mln::box2d{{0,0}, {h,w}});
}
m_filename = filename;
m_nbframe = 0;
......@@ -237,21 +236,62 @@ public:
}
}
std::array<mln::point2d, 4>
getquad(middle_filter_result* mdr) const
{
int selection = 0;
if (not m_results->empty() and m_results->back().good)
{
auto lastr = m_results->back().quad;
for (int i = 0; i < 3; ++i)
{
bool dismiss = false;
for (int j = 0; j < 4; ++j)
if (l2dist(mdr->res[i].points[j], lastr[j]) > MAX_DISTANCE_INTER_FRAME) {
dismiss = true;
break;
}
if (not dismiss) {
selection = i;
break;
}
}
}
std::cout << "### FRAME: " << m_nbframe << std::endl;
for (int i = 0; i < 3; ++i) {
std::cout << "\t" << (selection == i ? '*' : ' ')
<< "Quad " << i << ": " << mdr->res[i].energy << " "
<< mdr->res[i].points[0]
<< mdr->res[i].points[1]
<< mdr->res[i].points[3]
<< mdr->res[i].points[2]
<< std::endl;
}
return mdr->res[selection].points;
}
void
operator() (middle_filter_result* mdr) const
{
std::array<mln::point2d, 4> quad = this->getquad(mdr);
//mln::io::imsave(mdr->depth, "depth.tiff");
if (VIDEO_OUTPUT)
{
// Copy the image to the output buffer.
{
mln::image2d<mln::rgb8>& f = mdr->ima;
char* ptrin = (char*) &(f.at(0,0));
draw_quad_superimpose(quad, mdr->depth, m_output);
char* ptrin = (char*) &(m_output.at(0,0));
char* ptrout = (char*) pFrame_outRGB->data[0];
size_t stride = f.strides()[0];
for (unsigned i = 0; i < f.nrows(); ++i)
size_t stride = m_output.strides()[0];
for (unsigned i = 0; i < m_output.nrows(); ++i)
{
std::copy(ptrin, ptrin + (f.ncols() * sizeof(mln::rgb8)), ptrout);
std::copy(ptrin, ptrin + (m_output.ncols() * sizeof(mln::rgb8)), ptrout);
ptrin += stride;
ptrout += pFrame_outRGB->linesize[0];
}
......@@ -259,7 +299,7 @@ public:
const_cast<output_filter*>(this)->encode();
}
// Insert the smartdoc result in the vector.
m_results->push_back(mdr->res);
m_results->push_back({true, quad});
delete mdr;
}
......@@ -282,6 +322,7 @@ private:
struct SwsContext *convert_ctx_rgb2out;
// For other stuff
mutable mln::image2d<mln::rgb8> m_output;
const char* m_filename;
mutable int m_nbframe;
mutable int m_skipped;
......@@ -324,7 +365,7 @@ int main(int argc, char** argv)
tbb::filter_t<void,void> f = f1 & f2 & f3;
tbb::parallel_pipeline(8,f);
tbb::parallel_pipeline(4,f);
export_xml(output_path, input_path, &results[0], results.size());
}
......@@ -22,13 +22,14 @@ int main(int argc, char** argv)
tree_t tree;
morpho::load(argv[1], tree);
image2d<uint16> ima;
io::imread(argv[2], ima);
image2d<uint16> depth;
io::imread(argv[2], depth);
image2d<rgb8> out;
process(tree, ima, &out, argv[4], argv[3]);
std::array<process_result_t, 3> res = process(tree, depth, argv[4], argv[3]);
resize(out, depth);
draw_quad_superimpose(res[0].points, depth, out);
io::imsave(out, argv[5]);
}
......@@ -3,8 +3,11 @@
#include <mln/io/imsave.hpp>
#include <mln/accu/accumulators/accu_as_it.hpp>
#include <mln/accu/accumulators/count.hpp>
#include <mln/accu/accumulators/max.hpp>
#include <mln/core/algorithm/transform.hpp>
#include <mln/core/algorithm/accumulate.hpp>
#include <mln/morpho/component_tree/io.hpp>
#include <mln/morpho/extinction.hpp>
#include <apps/tos/croutines.hpp>
......@@ -163,11 +166,12 @@ namespace mln
using namespace mln;
using vec2df = vec<float, 2>;
std::pair<bool, std::array<point2d, 4> >
std::array<process_result_t, 3>
process(tree_t& tree,
const image2d<uint16>& ima,
image2d<rgb8>* feedback,
const char* csvfile,
const char* treefile)
{
......@@ -175,169 +179,179 @@ process(tree_t& tree,
// Filter first
grain_filter_inplace(tree, GRAINSIZE);
property_map<tree_t, float> energy(tree);
property_map<tree_t, accu::accumulators::fitting_quad> quadri;
// 1. Compute energy related attributes
{
// Accumulation du nombre de feuilles + profondeurs.
property_map<tree_t, unsigned> nleaves(tree, 0);
property_map<tree_t, unsigned> sumdepth(tree, 0);
property_map<tree_t, unsigned> sumdepth2(tree, 0);
property_map<tree_t, unsigned> depth = morpho::compute_depth(tree);
property_map<tree_t, unsigned> area = morpho::accumulate(tree, accu::features::count<unsigned> ());
float maxdepth = 0;
mln_reverse_foreach(auto x, tree.nodes())
{
if (not x.has_child())
{
nleaves[x]++;
sumdepth[x] += depth[x];
}
if (depth[x] > maxdepth)
maxdepth = depth[x];
nleaves[x.parent()] += nleaves[x];
sumdepth[x.parent()] += sumdepth[x];
sumdepth2[x] = sumdepth[x] - (depth[x] * nleaves[x]);
}
// Accumulation du nombre de feuilles + profondeurs.
property_map<tree_t, unsigned> nleaves(tree, 0);
property_map<tree_t, unsigned> sumdepth(tree, 0);
property_map<tree_t, unsigned> sumdepth2(tree, 0);
property_map<tree_t, unsigned> depth = morpho::compute_depth(tree);
property_map<tree_t, unsigned> area = morpho::accumulate(tree, accu::features::count<unsigned> ());
typedef accu::accumulators::accu_as_it<accu::accumulators::fitting_quad> accu_t;
quadri = morpho::paccumulate(tree, ima, accu_t ());
float maxdepth = 0;
mln_reverse_foreach(auto x, tree.nodes())
// Recompute the point inside/outside
mln_pixter(px, ima);
mln_forall(px)
{
if (not x.has_child())
tree_t::node_type x = tree.get_node_at(px->index());
vec2df p_ = px->point().as_vec();
while (x.id() != tree.npos())
{
nleaves[x]++;
sumdepth[x] += depth[x];
vec2df p = p_ - quadri[x].m_quad[0].as_vec();
vec2df u = (quadri[x].m_quad[1] - quadri[x].m_quad[0]).as_vec();
vec2df v = (quadri[x].m_quad[2] - quadri[x].m_quad[0]).as_vec();
vec2df w = (quadri[x].m_quad[3] - quadri[x].m_quad[0]).as_vec();
bool inside1, inside2;
{
float n = (u[0]*v[1] - u[1]*v[0]);
float alpha = (u[0]*p[1] - u[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside1 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
{
float n = (w[0]*v[1] - w[1]*v[0]);
float alpha = (w[0]*p[1] - w[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside2 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
quadri[x].m_inside += inside1 or inside2;
x = x.parent();
}
if (depth[x] > maxdepth)
maxdepth = depth[x];
nleaves[x.parent()] += nleaves[x];
sumdepth[x.parent()] += sumdepth[x];
sumdepth2[x] = sumdepth[x] - (depth[x] * nleaves[x]);
}
typedef accu::accumulators::accu_as_it<accu::accumulators::fitting_quad> accu_t;
auto quadri = morpho::paccumulate(tree, ima, accu_t ());
// Recompute the point inside/outside
using vec2df = vec<float, 2>;
auto noising = make_functional_property_map<tree_t::node_type>
([&sumdepth2,&area](const tree_t::node_type& x) {
return (sumdepth2[x] / float(area[x]));
});
mln_pixter(px, ima);
mln_forall(px)
{
tree_t::node_type x = tree.get_node_at(px->index());
vec2df p_ = px->point().as_vec();
while (x.id() != tree.npos())
{
vec2df p = p_ - quadri[x].m_quad[0].as_vec();
vec2df u = (quadri[x].m_quad[1] - quadri[x].m_quad[0]).as_vec();
vec2df v = (quadri[x].m_quad[2] - quadri[x].m_quad[0]).as_vec();
vec2df w = (quadri[x].m_quad[3] - quadri[x].m_quad[0]).as_vec();
bool inside1, inside2;
{
float n = (u[0]*v[1] - u[1]*v[0]);
float alpha = (u[0]*p[1] - u[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside1 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
float imsize = ima.domain().size();
{
mln_foreach (const tree_t::node_type& x, tree.nodes())
{
float n = (w[0]*v[1] - w[1]*v[0]);
float alpha = (w[0]*p[1] - w[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside2 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
if (area[x] < (MIN_OBJECT_SIZE * imsize) or
area[x] > (MAX_OBJECT_SIZE * imsize))
energy[x] = 0;
else
energy[x] = WEIGHT_QUAD * quadri[x].to_result() + WEIGHT_NOISE * noising[x];
}
}
quadri[x].m_inside += inside1 or inside2;
x = x.parent();
// Save tree
if (treefile)
morpho::save(tree, treefile);
// Save attributes
if (csvfile)
{
std::ofstream f(csvfile);
f << "nleaves,sumdepth,noising,quadri,inside,energy,bbox,quad\n";
mln_foreach(auto x, tree.nodes())
f << nleaves[x]
<< "," << sumdepth[x]
<< "," << noising[x]
<< "," << quadri[x].to_result()
<< "," << (quadri[x].m_inside / float(quadri[x].m_count))
<< "," << energy[x]
<< "," << "\"" << quadri[x].m_pmin << quadri[x].m_pmax << "\""
<< "," << "\"" << quadri[x].m_quad[0] << quadri[x].m_quad[1] << quadri[x].m_quad[2] << quadri[x].m_quad[3] << "\""
<< "\n";
}
}
// 2. Compute extinction values
property_map<tree_t, float> extvalues;
{
auto eimg = morpho::make_image(tree, energy);
auto extimg = morpho::extinction(eimg, morpho::tree_neighb_t (), std::greater<float> ());
extvalues = std::move(extimg.get_vmap());
}
// Save tree
if (treefile)
morpho::save(tree, treefile);
// {
// accu::accumulators::fitting_quad test;
// test.take({1,2});
// test.take({2,6});
// test.take({9,1});
// test.take({8,7});
// std::cout << test.m_pmin << test.m_pmax << "\n";
// for (int i = 0; i < 4; ++i)
// std::cout << test.m_quad[i];
// std::cout << "\n" << test.to_result() << std::endl;
// }
auto noising = make_functional_property_map<tree_t::node_type>
([&sumdepth2,&area](const tree_t::node_type& x) {
return (sumdepth2[x] / float(area[x]));
});
float imsize = ima.domain().size();
auto energy = make_functional_property_map<tree_t::node_type>
([&quadri,&noising,&area,&depth,imsize,maxdepth](const tree_t::node_type& x) -> float{
if (area[x] < (MIN_OBJECT_SIZE * imsize) or
area[x] > (MAX_OBJECT_SIZE * imsize))
return 0;
return WEIGHT_QUAD * quadri[x].to_result() + WEIGHT_NOISE * noising[x];
});
// Save attributes
if (csvfile)
{
std::ofstream f(csvfile);
f << "nleaves,sumdepth,noising,quadri,inside,energy,bbox,quad\n";
mln_foreach(auto x, tree.nodes())
f << nleaves[x]
<< "," << sumdepth[x]
<< "," << noising[x]
<< "," << quadri[x].to_result()
<< "," << (quadri[x].m_inside / float(quadri[x].m_count))
<< "," << energy[x]
<< "," << "\"" << quadri[x].m_pmin << quadri[x].m_pmax << "\""
<< "," << "\"" << quadri[x].m_quad[0] << quadri[x].m_quad[1] << quadri[x].m_quad[2] << quadri[x].m_quad[3] << "\""
<< "\n";
}
//
tree_t::node_type shp = tree.get_root();
// 3. Get the top three shapes
tree_t::node_type shps[3] = {tree.get_root(), tree.get_root(), tree.get_root()};
{
float emax = value_traits<float>::min();
float emax[3] = {0,0,0};
mln_foreach(auto x, tree.nodes())
{
float e = energy[x];
if (e > emax) {
emax = e;
shp = x;
float e = extvalues[x];
if (e > emax[0]) {
shps[2] = shps[1]; shps[1] = shps[0]; shps[0] = x;
emax[2] = emax[1]; emax[1] = emax[0]; emax[0] = e;
} else if (e > emax[1]) {
shps[2] = shps[1]; shps[1] = x;
emax[2] = emax[1]; emax[1] = e;
} else if (e > emax[2]) {
shps[2] = x;
emax[2] = e;
}
}
}
if (feedback)
std::array<process_result_t, 3> res;
for (int i = 0; i < 3; ++i) {
std::copy(quadri[shps[i]].m_quad, quadri[shps[i]].m_quad + 4, res[i].points.begin());
res[i].energy = energy[shps[i]];
}
return res;
}
void
draw_quad_superimpose(std::array<mln::point2d, 4> quad,
const image2d<uint16>& depth,
image2d<rgb8>& out)
{
float maxdepth = accumulate(depth, accu::features::max<>());
transform(depth, [maxdepth](uint16 x) {
uint8 v = std::min(1.0f, (x / maxdepth)) * 127;
return rgb8(v);
}, out);
vec2df u = (quad[1] - quad[0]).as_vec();
vec2df v = (quad[2] - quad[0]).as_vec();
vec2df w = (quad[3] - quad[0]).as_vec();
float n1 = (u[0]*v[1] - u[1]*v[0]);
float n2 = (w[0]*v[1] - w[1]*v[0]);
mln_foreach(point2d x, out.domain())
{
image2d<rgb8> out = transform(ima, [maxdepth](uint16 x) {
uint8 v = std::min(1.0f, (x / maxdepth)) * 127;
return rgb8(v);
});
vec2df u = (quadri[shp].m_quad[1] - quadri[shp].m_quad[0]).as_vec();
vec2df v = (quadri[shp].m_quad[2] - quadri[shp].m_quad[0]).as_vec();
vec2df w = (quadri[shp].m_quad[3] - quadri[shp].m_quad[0]).as_vec();
float n1 = (u[0]*v[1] - u[1]*v[0]);
float n2 = (w[0]*v[1] - w[1]*v[0]);
mln_foreach(point2d x, ima.domain())
{
vec2df p = x.as_vec() - quadri[shp].m_quad[0].as_vec();
vec2df p = x.as_vec() - quad[0].as_vec();
bool inside1, inside2;
{
float alpha = (u[0]*p[1] - u[1]*p[0]) / n1;
float beta = (p[0]*v[1] - p[1]*v[0]) / n1;
inside1 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
{
float alpha = (w[0]*p[1] - w[1]*p[0]) / n2;
float beta = (p[0]*v[1] - p[1]*v[0]) / n2;
inside2 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
bool inside1, inside2;
{
float alpha = (u[0]*p[1] - u[1]*p[0]) / n1;
float beta = (p[0]*v[1] - p[1]*v[0]) / n1;
inside1 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
{
float alpha = (w[0]*p[1] - w[1]*p[0]) / n2;
float beta = (p[0]*v[1] - p[1]*v[0]) / n2;
inside2 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
if (inside1 or inside2)
out(x)[0] += 128;
}
std::cout << out.domain() << feedback->domain() << std::endl;
copy(out, *feedback);
if (inside1 or inside2)
out(x)[0] += 128;
}
std::array<point2d, 4> res;
std::copy(quadri[shp].m_quad, quadri[shp].m_quad + 4, res.begin());
return std::make_pair(true, res);
}
......@@ -22,12 +22,20 @@ constexpr float WEIGHT_DEPTH = 1;
constexpr float WEIGHT_QUAD = 1;
constexpr float WEIGHT_NOISE = 1;
constexpr float MAX_DISTANCE_INTER_FRAME = 100;
using tree_t = mln::morpho::component_tree<unsigned, mln::image2d<unsigned> >;
struct process_result_t
{
std::array<mln::point2d, 4> points;
float energy;
};
/// \brief Document detection method entry point.
/// It returns the pair \p (detected, QUAD) where \p detected
......@@ -39,12 +47,18 @@ using tree_t = mln::morpho::component_tree<unsigned, mln::image2d<unsigned> >;
/// \param[out] feedback The image with the detected object colorized (must be the same size of imdepth)
/// \param csvfile (optional) Path to the CSV file where attributes will be saved
/// \param treefile (optional) Path to the file where the tree will be saved (after filtering)
std::pair<bool, std::array<mln::point2d, 4> >
std::array<process_result_t, 3>
process(tree_t& tree,
const mln::image2d<mln::uint16>& imdepth,
mln::image2d<mln::rgb8>* feedback = nullptr,
const char* csvfile = nullptr,
const char* treefile = nullptr);
void
draw_quad_superimpose(std::array<mln::point2d, 4> quad,
const mln::image2d<mln::uint16>& depth,