Commit c33f9015 authored by Edwin Carlinet's avatar Edwin Carlinet
Browse files

Populate the smartdoc application.

         *  apps/CMakeLists.txt,
         *  apps/smartdoc/CMakeLists.txt,
         *  apps/smartdoc/compare_xml.py,
         *  apps/smartdoc/export.cpp,
         *  apps/smartdoc/export.hpp,
         *  apps/smartdoc/main.cpp,
         *  apps/smartdoc/smartdoc-cli.cpp,
         *  apps/smartdoc/smartdoc.cpp,
         *  apps/smartdoc/smartdoc.hpp,
         *  apps/smartdoc/video_tools.cpp,
         *  apps/smartdoc/video_tools.hh: New.
parent 825a7819
...@@ -12,6 +12,7 @@ add_subdirectory(misc) ...@@ -12,6 +12,7 @@ add_subdirectory(misc)
add_subdirectory(theo) add_subdirectory(theo)
add_subdirectory(tosgui) add_subdirectory(tosgui)
add_subdirectory(saliency) add_subdirectory(saliency)
add_subdirectory(smartdoc)
add_subdirectory(supervised-gui) add_subdirectory(supervised-gui)
add_subdirectory(hierachical_seg-gui) add_subdirectory(hierachical_seg-gui)
add_subdirectory(hyperspec) add_subdirectory(hyperspec)
add_executable(smartdoc_app main.cpp
video_tools.cpp
smartdoc.cpp
export.cpp
../g2/compute_g2.cpp
../g2/compute_ctos.cpp
../g2/satmaxtree.cpp
)
add_executable(smartdoc_cli smartdoc-cli.cpp smartdoc.cpp)
target_link_libraries(smartdoc_cli ${FreeImage_LIBRARIES})
target_link_libraries(smartdoc_app
-lavcodec -lavformat -lswscale
${FreeImage_LIBRARIES})
set_source_files_properties(video_tools.cpp PROPERTIES
COMPILE_DEFINITIONS __STDC_CONSTANT_MACROS)
#target_set_properties(
\ No newline at end of file
#! /usr/bin/python
import sys
import xml.etree.ElementTree as ET
if len(sys.argv) != 3:
print "Usage:", sys.argv[0], "fichier1.xml fichier2.xml"
tol = 4
tree1 = ET.parse(sys.argv[1])
tree2 = ET.parse(sys.argv[2])
frames1 = tree1.getroot().findall("*/frame")
frames2 = tree2.getroot().findall("*/frame")
def getPoint(point):
x = float(point.get("x"))
y = float(point.get("y"))
return (x,y)
def getPoints(frame):
tl = getPoint(frame.find("./point[@name='tl']"))
tr = getPoint(frame.find("./point[@name='tr']"))
bl = getPoint(frame.find("./point[@name='bl']"))
br = getPoint(frame.find("./point[@name='br']"))
return (tl, tr, bl, br)
def checkCoord(f1, f2, idx):
pts1 = getPoints(f1)
pts2 = getPoints(f2)
s = 0
for p1, p2 in zip(pts1, pts2):
s += ((p1[0] - p2[0])**2 + (p1[1] - p2[1])**2) ** .5
print "Frame", idx, "error:", s
for f1, f2 in zip(frames1, frames2):
assert f1.get("index") == f2.get("index")
idx = f1.get("index")
if f1.get("rejected") != f2.get("rejected"):
print "Frame", idx, ": rejection error"
else:
checkCoord(f1, f2, idx)
#include "export.hpp"
#include <iostream>
#include <fstream>
#include <chrono>
#include <boost/date_time/posix_time/posix_time.hpp>
void export_xml(const char* filename,
const char* document_id,
const SmartDoc_t* p_array,
int p_array_size)
{
using namespace boost::posix_time;
std::ofstream f(filename);
ptime now = second_clock::local_time();
f << "<?xml version='1.0' encoding='utf-8'?>\n"
<< "<seg_result version=\"0.2\" generated=\"" << to_iso_string(now) << "\">\n"
<< " <software_used name=\"" << APP_NAME << "\" version=\"" << APP_VERSION << "\"/>\n"
<< " <source_sample_file>" << document_id << "</source_sample_file>\n"
<< " <segmentation_results>\n"
;
// K0 + border
auto p_cvt = [](int i) -> float { return i / 2.0 - 1; };
for (int i = 0; i < p_array_size; ++i)
{
SmartDoc_t x = p_array[i];
f << " <frame index=\"" << (i+1) << "\" rejected=\"" << std::boolalpha << (not x.good) << "\">\n";
if (x.good)
{
f <<
" <point name=\"tl\" x=\"" << p_cvt(x.quad[0][1]) << "\" y=\"" << p_cvt(x.quad[0][0]) << "\"/>\n"
" <point name=\"tr\" x=\"" << p_cvt(x.quad[1][1]) << "\" y=\"" << p_cvt(x.quad[1][0]) << "\"/>\n"
" <point name=\"br\" x=\"" << p_cvt(x.quad[2][1]) << "\" y=\"" << p_cvt(x.quad[2][0]) << "\"/>\n"
" <point name=\"bl\" x=\"" << p_cvt(x.quad[3][1]) << "\" y=\"" << p_cvt(x.quad[3][0]) << "\"/>\n"
;
}
f << " </frame>\n";
}
f << " </segmentation_results>\n"
<< "</seg_result>\n";
}
#ifndef APPS_SMARTDOC_EXPORT_HPP
# define APPS_SMARTDOC_EXPORT_HPP
# include <mln/core/point.hpp>
# include <array>
static const char* APP_NAME = "MILENA";
static const char* APP_VERSION = "2.0";
struct SmartDoc_t
{
bool good;
std::array<mln::point2d, 4> quad;
};
void
export_xml(const char* filename,
const char* document_id,
const SmartDoc_t* p_array,
int p_array_size);
#endif // ! APPS_SMARTDOC_EXPORT_HPP
#include <mln/core/image/image2d.hpp>
#include <mln/core/colors.hpp>
#include <mln/core/trace.hpp>
#include <mln/io/imsave.hpp>
#include <boost/format.hpp>
#include <apps/g2/compute_ctos.hpp>
#include "video_tools.hh"
#include "smartdoc.hpp"
#include "export.hpp"
/****************************************************/
/** CONFIG ***/
/****************************************************/
const bool ENABLE_CACHING = true;
int main(int argc, char** argv)
{
if (argc < 3) {
std::cerr << "Usage: " << argv[0] << " input.mpg output.xml [output.mpg]\n";
std::exit(1);
}
const char* input_path = argv[1];
const char* output_path = argv[2];
const char* output_video_path = (argc > 3) ? argv[3] : NULL;
bool VIDEO_OUTPUT = argc > 3;
using namespace mln;
/***********************************************************************/
/** De/en-coding loop with ffmpeg **/
/***********************************************************************/
// Initialization of the library
AVPacket packet_dec, packet_enc;
av_register_all();
avcodec_register_all();
av_init_packet(&packet_dec);
// Get info about input/output videos
int videoStream = -1;
AVFrame *pFrame_inYUV, *pFrame_inRGB, *pFrame_outYUV, *pFrame_outRGB;
AVCodecContext *Ctx_Enc, *Ctx_Dec;
AVFormatContext *pFormatCtx_Dec, *pFormatCtx_Enc;
pFormatCtx_Dec = init_format_context_decode(trace::verbose,
input_path,
videoStream, &pFrame_inYUV, &pFrame_inRGB,
&Ctx_Dec);
if (VIDEO_OUTPUT) {
pFormatCtx_Enc = init_format_context_encode(trace::verbose,
output_video_path,
Ctx_Dec, pFormatCtx_Dec,
&pFrame_outYUV, NULL, &Ctx_Enc);
// Because K0 -> bug alignement
int w = (pFrame_inRGB->width + 2) * 2 - 1;
int h = (pFrame_inRGB->height + 2) * 2 - 1;
int ws = (w * 3);
// Si on est sur 96 octets (aligné sur 32 octets et 3 octets en même temps OK)
// Sinon on aloue un peu plus
if (ws % 96 != 0)
w = ((ws / 96) + 1) * (96 / 3);
pFrame_outRGB = avcodec_alloc_frame();
av_image_alloc(pFrame_outRGB->data, pFrame_outRGB->linesize,
w, h, PIX_FMT_RGB24, 32); // Car aligné sur 32 et 24 en même temps
pFrame_outRGB->width = w;
pFrame_outRGB->height = h;
pFrame_outRGB->format = PIX_FMT_RGB24;
}
// Main decoding loop
{
// Definition of milena types (they use external memory management layout)
image2d<rgb8> img_in, img_out;
{
size_t strides[2] = {(size_t)pFrame_inRGB->linesize[0], sizeof(rgb8)};
img_in = image2d<rgb8>::from_buffer(pFrame_inRGB->data[0],
box2d{{0,0},{(short)pFrame_inRGB->height, (short)pFrame_inRGB->width}},
strides);
}
if (VIDEO_OUTPUT) {
short w = (pFrame_inRGB->width + 2) * 2 - 1;
short h = (pFrame_inRGB->height + 2) * 2 - 1;
// std::cout << "Width: " << pFrame_outRGB->width << "(real: " << w << ")" << std::endl;
// std::cout << pFrame_outRGB->linesize[0] << std::endl;
size_t strides[2] = {(size_t)pFrame_outRGB->linesize[0], sizeof(rgb8)};
img_out = image2d<rgb8>::from_buffer(pFrame_outRGB->data[0],
box2d{{0,0},{h, w}},
strides);
}
// Defintion of contexts from YUV-RGB conversion and K0 desimmersion
struct SwsContext *convert_ctx_yuv2rgb, *convert_ctx_rgb2out;
convert_ctx_yuv2rgb = sws_getContext(Ctx_Dec->width,
Ctx_Dec->height,
Ctx_Dec->pix_fmt,
Ctx_Dec->width,
Ctx_Dec->height,
PIX_FMT_RGB24,
SWS_BILINEAR, 0, 0, 0);
if (VIDEO_OUTPUT) {
convert_ctx_rgb2out = sws_getContext(pFrame_outRGB->width,
pFrame_outRGB->height,
PIX_FMT_RGB24,
Ctx_Enc->width,
Ctx_Enc->height,
Ctx_Enc->pix_fmt,
SWS_BILINEAR, 0, 0, 0);
}
tree_t tree;
image2d<uint16> depth;
int frameFinished = 0;
int nbframe = 0;
int skipped = 0; // ???
std::vector<SmartDoc_t> results;
image2d<rgb8>* ptr_img_out = (VIDEO_OUTPUT) ? &img_out : NULL;
std::cout << "Decoding: " << std::endl;
while (av_read_frame(pFormatCtx_Dec, &packet_dec) >= 0)
if (packet_dec.stream_index == videoStream)
{
avcodec_decode_video2(Ctx_Dec, pFrame_inYUV, &frameFinished, &packet_dec);
if (frameFinished)
{
std::cout << "Processing #" << nbframe << std::endl;
/* convert from YUV to RGB24 to decode video */
sws_scale(convert_ctx_yuv2rgb,
(const uint8_t * const*) pFrame_inYUV->data,
pFrame_inYUV->linesize,
0,
Ctx_Dec->height,
pFrame_inRGB->data,
pFrame_inRGB->linesize);
// Process
SmartDoc_t res;
tree = compute_ctos(img_in, &depth);
std::tie(res.good, res.quad) = process(tree, depth, ptr_img_out);
// Output the video
if (VIDEO_OUTPUT) {
encode_video(skipped, &pFormatCtx_Enc, output_video_path,
&pFrame_outRGB, &pFrame_outYUV, nbframe,
&Ctx_Enc, packet_enc, &convert_ctx_rgb2out);
io::imsave(img_out, (boost::format("out-%03d.png") % nbframe).str());
}
// Write results.
results.push_back(res);
nbframe++;
}
}
export_xml(output_path, input_path, &results[0], results.size());
if (VIDEO_OUTPUT) {
encode_skipped_frames(skipped, nbframe, &pFormatCtx_Enc, output_video_path, &pFrame_outYUV, &Ctx_Enc, packet_enc);
close_free_encode(&pFormatCtx_Enc, &Ctx_Enc, &pFrame_outYUV, &pFrame_outRGB);
}
}
close_free_decode(&pFormatCtx_Dec, &Ctx_Dec, &pFrame_inYUV, &pFrame_inRGB);
}
#include <mln/core/image/image2d.hpp>
#include <mln/morpho/component_tree/io.hpp>
#include <mln/io/imread.hpp>
#include <mln/io/imsave.hpp>
#include "smartdoc.hpp"
int main(int argc, char** argv)
{
if (argc < 6)
{
std::cerr << "Usage: " << argv[0] << "input.tree input.tiff out.tree out.csv out.tiff\n"
;
std::exit(1);
}
using namespace mln;
tree_t tree;
morpho::load(argv[1], tree);
image2d<uint16> ima;
io::imread(argv[2], ima);
image2d<rgb8> out;
process(tree, ima, &out, argv[4], argv[3]);
io::imsave(out, argv[5]);
}
#include "smartdoc.hpp"
#include <mln/core/algorithm/copy.hpp>
#include <mln/io/imsave.hpp>
#include <mln/accu/accumulators/accu_as_it.hpp>
#include <mln/accu/accumulators/count.hpp>
#include <mln/core/algorithm/transform.hpp>
#include <mln/morpho/component_tree/io.hpp>
#include <apps/tos/croutines.hpp>
namespace mln
{
template <typename T>
using vec2d = vec<T, 2>;
template <typename T>
float
mydist(vec2d<T> x, vec2d<T> y)
{
int a = abs(y[0] - x[0]);
int b = abs(y[1] - x[1]);
int i,j;
std::tie(i,j) = std::minmax(a,b);
return i * 2 + (j-i) * 1;
}
namespace accu
{
namespace accumulators
{
struct fitting_quad : accumulator_base<fitting_quad, point2d, float, void>
{
typedef point2d argument_type;
typedef float result_type;
fitting_quad()
{
for (int i = 0; i < 4; ++i)
m_quad[i] = value_traits<point2d>::sup();
m_pmin = value_traits<point2d>::sup();
m_pmax = value_traits<point2d>::inf();
m_count = 0;
m_inside = 0;
}
void init()
{
for (int i = 0; i < 4; ++i)
m_quad[i] = value_traits<point2d>::sup();
m_pmin = value_traits<point2d>::sup();
m_pmax = value_traits<point2d>::inf();
m_count = 0;
m_inside = 0;
}
void take(point2d x)
{
m_pmin = inf(m_pmin, x);
m_pmax = sup(m_pmax, x);
vec2d<short> pts[4];
pts[0] = {m_pmin[0], m_pmin[1]};
pts[1] = {m_pmin[0], m_pmax[1]};
pts[2] = {m_pmax[0], m_pmax[1]};
pts[3] = {m_pmax[0], m_pmin[1]};
for (int i = 0; i < 4; ++i)
{
float d1 = mydist(pts[i], (vec2d<short>)m_quad[i]);
float d2 = mydist(pts[i], (vec2d<short>)x);
if (d2 < d1) {
m_quad[i] = x;
}
}
m_count++;
}
void take(const fitting_quad& other)
{
m_pmin = inf(m_pmin, other.m_pmin);
m_pmax = sup(m_pmax, other.m_pmax);
vec2d<short> pts[4];
pts[0] = {m_pmin[0], m_pmin[1]};
pts[1] = {m_pmin[0], m_pmax[1]};
pts[2] = {m_pmax[0], m_pmax[1]};
pts[3] = {m_pmax[0], m_pmin[1]};
for (int i = 0; i < 4; ++i)
{
float d1 = mydist(pts[i], (vec2d<short>)m_quad[i]);
float d2 = mydist(pts[i], (vec2d<short>)other.m_quad[i]);
if (d1 < d2) {
m_quad[i] = m_quad[i];
} else {
m_quad[i] = other.m_quad[i];
}
}
m_count += other.m_count;
}
friend
unsigned
extract(const fitting_quad& accu, accu::features::count<>)
{
return accu.m_count;
}
float to_result() const
{
// Si les points du quadri sont distants de plus de 5%
// de bbbox, c'est vain
float scale = 1.0 / mydist(m_pmin.as_vec(), m_pmax.as_vec());
vec2d<short> pts[4];
pts[0] = {m_pmin[0], m_pmin[1]};
pts[1] = {m_pmin[0], m_pmax[1]};
pts[2] = {m_pmax[0], m_pmax[1]};
pts[3] = {m_pmax[0], m_pmin[1]};
// float sumdist = 0;
// for (int i = 0; i < 4; ++i) {
// sumdist += l2dist_sqr(m_quad[i], (point2df)pts[i]) * scale;
// }
// return sumdist;
for (int i = 0; i < 4; ++i) {
float d = mydist((vec2d<short>)m_quad[i], pts[i]) * scale;
if (d > CORNER_TOLERANCE)
return 0;
}
// Calcul de l'aire du //gramme
// Somme des cross products entre les vertexes
float s = 0.0;
for (int i = 0; i < 3; ++i)
s += m_quad[i][0] * m_quad[i+1][1] - m_quad[i+1][0] * m_quad[i][1];
s += m_quad[3][0] * m_quad[0][1] - m_quad[0][0] * m_quad[3][1];
float area = (0.5 * abs(s));
float m_outside = m_count - m_inside;
return (0.8 * m_inside - 1.2 * m_outside) / area;
}
public:
unsigned m_count;
unsigned m_inside;
point2d m_pmin, m_pmax;
point2df m_quad[4];
};
}
}
}
using namespace mln;
std::pair<bool, std::array<point2d, 4> >
process(tree_t& tree,
const image2d<uint16>& ima,
image2d<rgb8>* feedback,
const char* csvfile,
const char* treefile)
{
// Filter first
grain_filter_inplace(tree, GRAINSIZE);
// Accumulation du nombre de feuilles + profondeurs.
property_map<tree_t, unsigned> nleaves(tree, 0);
property_map<tree_t, unsigned> sumdepth(tree, 0);
property_map<tree_t, unsigned> sumdepth2(tree, 0);
property_map<tree_t, unsigned> depth = morpho::compute_depth(tree);
property_map<tree_t, unsigned> area = morpho::accumulate(tree, accu::features::count<unsigned> ());
float maxdepth = 0;
mln_reverse_foreach(auto x, tree.nodes())
{
if (not x.has_child())
{
nleaves[x]++;
sumdepth[x] += depth[x];
}
if (depth[x] > maxdepth)
maxdepth = depth[x];
nleaves[x.parent()] += nleaves[x];
sumdepth[x.parent()] += sumdepth[x];
sumdepth2[x] = sumdepth[x] - (depth[x] * nleaves[x]);
}
typedef accu::accumulators::accu_as_it<accu::accumulators::fitting_quad> accu_t;
auto quadri = morpho::paccumulate(tree, ima, accu_t ());
// Recompute the point inside/outside
using vec2df = vec<float, 2>;
mln_pixter(px, ima);
mln_forall(px)
{
tree_t::node_type x = tree.get_node_at(px->index());
vec2df p_ = px->point().as_vec();
while (x.id() != tree.npos())
{
vec2df p = p_ - quadri[x].m_quad[0].as_vec();
vec2df u = (quadri[x].m_quad[1] - quadri[x].m_quad[0]).as_vec();
vec2df v = (quadri[x].m_quad[2] - quadri[x].m_quad[0]).as_vec();
vec2df w = (quadri[x].m_quad[3] - quadri[x].m_quad[0]).as_vec();
bool inside1, inside2;
{
float n = (u[0]*v[1] - u[1]*v[0]);
float alpha = (u[0]*p[1] - u[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside1 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
{
float n = (w[0]*v[1] - w[1]*v[0]);
float alpha = (w[0]*p[1] - w[1]*p[0]) / n;
float beta = (p[0]*v[1] - p[1]*v[0]) / n;
inside2 = 0 <= alpha and 0 <= beta and (alpha + beta) <= 1;
}
quadri[x].m_inside += inside1 or inside2;
x = x.parent();
}
}