MPEG CDVA Experimentation Model (CXM)  1.0
Compact Descriptors for Visual Analisys
CdvaImpl.h
Go to the documentation of this file.
1 /*
2 The copyright in this software is being made available under this MPEG Reference Software Copyright License. This software may be subject to other third party and contributor rights, including patent rights, and no such rights are granted under this license.
3 
4 Copyright (c) 2016-2017, Joanneum Research, Mitsubishi Electric Research Labs, Peking University, Telecom Italia, University of Surrey, Visual Atoms
5 
6 Redistribution and use in source and binary forms, with or without modification, are permitted provided that the following conditions are met:
7 * Redistributions of source code must retain the above copyright notice, this list of conditions and the following disclaimer.
8 * Redistributions in binary form must reproduce the above copyright notice, this list of conditions and the following disclaimer in the documentation and/or other materials provided with the distribution.
9 * Neither the names of Joanneum Research, Mitsubishi Electric Research Labs, Peking University, Telecom Italia, University of Surrey, Visual Atoms nor the names of their contributors may be used to endorse or promote products derived from this software without specific prior written permission.
10 THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
11 */
12 #pragma once
13 
14 #include <string>
15 #include <vector>
16 #include <map>
17 
18 #include "cdva.h"
19 #include "CdvaException.h"
20 #include "LogManager.h"
21 #include "Buffer.h"
22 #include "CdvsInterface.h"
23 
24 #include "SCFVIndex.h"
25 #include "DescriptorTimeMap.h"
26 #include "CompressedFeatureList.h"
27 
33 namespace mpeg7cdva
34 {
35 
37 
38 
44  {
45 
46  protected:
47  unsigned long startTime;
48  unsigned long endTime;
49  unsigned int parity;
50 
51  static const int SHOT_HEADER_SIZE = 10; // header size in bytes
52 
53  size_t getSize() const; // get the size in bytes of encoded keyframes when written into a buffer
54 
55  public:
57  virtual ~SegmentDescriptor();
58 
59  std::vector <mpeg7cdvs::CdvsDescriptor> keyframes;
60 
61  void setParity(unsigned int value);
62  unsigned int getParity() const;
63  void setStartTimeMs(unsigned long position_ms);
64  void setEndTimeMs(unsigned long position_ms);
65  unsigned long getStartTimeMs() const;
66  unsigned long getEndTimeMs() const;
67  virtual size_t write(std::ofstream & fout) const;
68  virtual size_t read(std::ifstream & fin, mpeg7cdvs::CdvsServer * cdvsserver);
69  static size_t addSegmentToDB(std::ifstream & fin, mpeg7cdvs::CdvsServer * cdvsserver, const std::string & relativepathname,bool optMatch);
70  bool empty() const;
71  virtual void clear();
72  };
73 
79  {
80  protected:
81  unsigned char* globalBuf;
82  unsigned char* localBuf;
83  unsigned char* histoBuf;
84  unsigned long globalBufSz;
85  unsigned long localBufSz;
86  unsigned long histoBufSz;
87  unsigned long globalUncompressedBufSz;
88  unsigned long localUncompressedBufSz;
89 
91 
92  const mpeg7cdvs::Parameters& params;
93 
94  static const int SHOT_HEADER_SIZE = 35; // header size in bytes
95 
96  void decode(char* globalMedBuf, unsigned int globalMedBufSz, std::vector<unsigned int>& nLocalDesc,
97  unsigned char cdvsDescriptorMode, unsigned short origImgX, unsigned short origImgY, unsigned short histoMapSzX, unsigned short histoMapSzY,
98  bool hasBitSel, bool hasVar, bool hasRelB);
99 
100  void reconstructGlobalDifferences(char* uncompressedData, unsigned int uncompressedSz, int nrGDesc, bool hasVar, bool globalHasBitSelection, std::vector<unsigned int>& nLocalDesc);
101 
102  mpeg7cdvs::FeatureList readFeatureListFromBinaryAbs(unsigned char* buffer, unsigned int bufferSz, bool readRelevance, unsigned int nrElemGr);
103 
104  inline int numGlobFctPresent(const mpeg7cdvs::CdvsDescriptor& desc) const {
105 
106  int numFctPresent = 0;
107  int offs = 8; // the interesting block starts at byte 8
108  for (int q = 0; q < 64; q++) {
109  numFctPresent += nBitsSet(desc.buffer.data()[offs + q]);
110 
111  }
112  return numFctPresent;
113  }
114 
115  inline int nBitsSet(unsigned char byte) const {
116  return ((byte & 0x01) ? 1 : 0) + ((byte & 0x02) ? 1 : 0) + ((byte & 0x04) ? 1 : 0) + ((byte & 0x08) ? 1 : 0) +
117  ((byte & 0x10) ? 1 : 0) + ((byte & 0x20) ? 1 : 0) + ((byte & 0x40) ? 1 : 0) + ((byte & 0x80) ? 1 : 0);
118  }
119 
120 
121  public:
122  CompressedSegmentDescriptor(const mpeg7cdvs::Parameters& params);
123  virtual ~CompressedSegmentDescriptor();
124 
125  virtual size_t write(std::ofstream & fout) const;
126  virtual size_t read(std::ifstream & fin, mpeg7cdvs::CdvsServer * cdvsserver);
127 
128  virtual unsigned char* allocateLocalBuffer(unsigned long sz);
129  virtual unsigned char* allocateGlobalBuffer(unsigned long sz);
130  virtual unsigned char* allocateHistoBuffer(unsigned long sz);
131 
132  virtual unsigned char* getLocalBuffer() { return localBuf; }
133  virtual unsigned char* getGlobalBuffer() { return globalBuf; }
134  virtual unsigned char* getHistoBuffer() { return histoBuf; }
135 
136  virtual unsigned long getLocalBufSz() { return localBufSz; }
137  virtual unsigned long getGlobalBufSz() { return globalBufSz; }
138  virtual unsigned long getHistoBufSz() { return histoBufSz; }
139 
140  virtual void setHistoBufSz(unsigned long sz) { histoBufSz = sz; }
141 
142  virtual void setLocalUncompressedBufSz(unsigned long sz) { localUncompressedBufSz = sz; }
143  virtual void setGlobalUncompressedBufSz(unsigned long sz) { globalUncompressedBufSz = sz; }
144 
145  virtual DescriptorTimeMap& getDtm() { return dtm; }
146 
147  virtual void clear();
148 
149  };
150 
151 
152  typedef std::vector<SegmentDescriptor> ShotDescriptorList;
153 
154 
155  class RefFeature;
156 
157 
162  class CdvaImpl
163  {
164  private:
165  void updateCounters(const mpeg7cdvs::CdvsDescriptor & desc, int total_bit_count, ExtractData & outdata) const;
166  protected:
167  mpeg7cdvs::CdvsConfiguration * cdvsconfig;
168  mpeg7cdvs::CdvsClient * cdvsclient;
169  mpeg7cdvs::CdvsServer * cdvsserver;
170 
171  bool verboseMode;
173  int cdvsMode;
176  double drop_frame_th;
177  double shot_cut_th;
178  double shot_ver_th;
179  double encode_th;
180  size_t max_retrieved;
186  bool optMatch;
188  double optMatch_tau;
189  double optMatch_b;
190 
191  static bool byDescendingScore(const MatchData & m1, const MatchData & m2); // order data by descending score
192 
193  virtual void parse(const std::string & descFile, ShotDescriptorList& shotList);
194 
195  virtual double match(MatchData & matchResults, const ShotDescriptorList& qDescList, const ShotDescriptorList& rDescList);
196 
200  virtual double match_med1(MatchData & matchResults, const ShotDescriptorList& qDescList, const ShotDescriptorList& rDescList);
201 
205  virtual double match_med2(MatchData & matchResults, const ShotDescriptorList& qDescList, const ShotDescriptorList& rDescList);
206 
207 
208  virtual int encodeShot(SegmentDescriptor& shot, unsigned long endTime, mpeg7cdvs::CdvsDescriptor& medoid, std::ofstream & fout, ExtractData & outdata) const; // returns nr frames encoded
209 
210  virtual void getDiffSignature(mpeg7cdvs::SCFVSignature& medoid, mpeg7cdvs::SCFVSignature& other, mpeg7cdvs::SCFVSignature& diffSig, unsigned int& bufferSize, unsigned char* buffer, unsigned int* globaldstatsBin) const;
211 
212 
216  virtual void localDescCodingAbs(SegmentDescriptor& shot, std::vector<int>& framesToCode, unsigned int& bufferSize, unsigned char* buffer, int medoidIdx, unsigned int* localdstats) const;
217 
220  void generateDTM(CompressedSegmentDescriptor& shot, std::vector<int>& framesToCode, int medoidIdx, std::vector<RefFeature> refFeatureList, std::vector<int> ldOffsets, std::vector<int> localFrameOrder, int totalNrFeatures) const;
221 
222 
225  virtual void encodeCoordinates(CompressedSegmentDescriptor& shot, std::vector<int>& framesToCode, int medoidIdx, std::vector<RefFeature> refFeatureList, std::vector<int> ldOffsets) const;
226 
227 
228  static inline int nBitsSet(unsigned char byte) {
229  return ((byte & 0x01) ? 1 : 0) + ((byte & 0x02) ? 1 : 0) + ((byte & 0x04) ? 1 : 0) + ((byte & 0x08) ? 1 : 0) +
230  ((byte & 0x10) ? 1 : 0) + ((byte & 0x20) ? 1 : 0) + ((byte & 0x40) ? 1 : 0) + ((byte & 0x80) ? 1 : 0);
231  }
232 
233  static inline int nBitsSet(unsigned int word) {
234  unsigned char* ba = (unsigned char*)&word;
235  return nBitsSet(ba[0]) + nBitsSet(ba[1]) + nBitsSet(ba[2]) + nBitsSet(ba[3]);
236 
237  }
238 
239  /*
240  * Get the extension of a file name.
241  * @param imageName the original image/video name;
242  * @return the extension of the name;
243  */
244  static std::string getExt(const std::string & imageName);
245 
246  public:
252  static bool checkBitrate(int bitrate);
253 
259  static const char * getDescriptorExt(int bitrate);
260 
261  CdvaImpl();
262  virtual ~CdvaImpl();
263 
277  virtual void init(OPERATION op, bool verbose, size_t n_videos, int querybitrate, int refbitrate = 0, bool calcdescsizes = false, bool rwCompressed = false, bool optMatch = false, double set_drop_th = -1, double set_encode_th = -1);
278 
286  void extract(const std::string & descrname, const std::string & videopathname, int bitrate, ExtractData & outdata) const;
287 
297  virtual double match(MatchData & matchResults, const std::string & qdescrname, const std::string & rdescrname, int qbitrate, int rbitrate);
298 
304  virtual void makeindex(const std::string & cdva_descriptor, const std::string & relativepathname);
305 
312  virtual void retrieve(std::vector<MatchData> & retrievalResults, const std::string & qdescrname, int qbitrate);
313 
317  void commitDB();
318 
322  virtual void close();
323 
324  }; // end class CDVAImpl
325 
326 
327 } // end of namespace mpeg7cdva
virtual DescriptorTimeMap & getDtm()
Definition: CdvaImpl.h:145
int minLocalDiff
min local difference for lossy local descriptor coding (if diff is larger, descriptor is encoded...
Definition: CdvaImpl.h:184
OPERATION current_op
the current operation
Definition: CdvaImpl.h:172
unsigned int parity
Definition: CdvaImpl.h:49
int skip_before
number of video frames to skip before decoding one
Definition: CdvaImpl.h:174
static int nBitsSet(unsigned int word)
Definition: CdvaImpl.h:233
double shot_ver_th
shot verification threshold
Definition: CdvaImpl.h:178
OPERATION
Definition: CdvaImpl.h:36
double encode_th
threshold for encoding other frames than median
Definition: CdvaImpl.h:179
void setStartTimeMs(unsigned long position_ms)
Set the start time of the shot in milliseconds.
virtual unsigned long getGlobalBufSz()
Definition: CdvaImpl.h:137
A class containing the results of a matching or retrieval operation.
Definition: cdva.h:109
const mpeg7cdvs::Parameters & params
parameters needed to invoke histogram decoding
Definition: CdvaImpl.h:92
bool readWriteCompressed
read and write compressed descriptors
Definition: CdvaImpl.h:185
A CDVA implementation based on multiple CDVS descriptors.
Definition: CdvaImpl.h:162
unsigned long localUncompressedBufSz
size of unencoded local buffer
Definition: CdvaImpl.h:88
void setParity(unsigned int value)
Set the shot parity.
virtual unsigned long getHistoBufSz()
Definition: CdvaImpl.h:138
Definition: DescriptorTimeMap.h:24
int forceSampleMs
enforce a sample every x milliseconds
Definition: CdvaImpl.h:182
Namespace used to encapsulate all MPEG-7 CDVA declarations.
Definition: Buffer.h:14
int numGlobFctPresent(const mpeg7cdvs::CdvsDescriptor &desc) const
Definition: CdvaImpl.h:104
DescriptorTimeMap dtm
map for frame - descriptor association
Definition: CdvaImpl.h:90
int optMatch_mode
mode for optimised matching (1 or 2)
Definition: CdvaImpl.h:187
unsigned long histoBufSz
size of encoded histogram buffer
Definition: CdvaImpl.h:86
unsigned long getEndTimeMs() const
Get the frame start time in milliseconds.
virtual unsigned char * getHistoBuffer()
Definition: CdvaImpl.h:134
mpeg7cdvs::CdvsConfiguration * cdvsconfig
Definition: CdvaImpl.h:167
int minShotLen
minimum shot length
Definition: CdvaImpl.h:183
double shot_cut_th
shot cut threshold
Definition: CdvaImpl.h:177
unsigned char * histoBuf
buffer for histogram descriptors
Definition: CdvaImpl.h:83
mpeg7cdvs::CdvsClient * cdvsclient
Definition: CdvaImpl.h:168
int nBitsSet(unsigned char byte) const
Definition: CdvaImpl.h:115
virtual void setLocalUncompressedBufSz(unsigned long sz)
Definition: CdvaImpl.h:142
virtual unsigned char * getGlobalBuffer()
Definition: CdvaImpl.h:133
Definition: CdvaImpl.h:36
bool empty() const
returns true if the segment is empty
unsigned long localBufSz
size of encoded local buffer
Definition: CdvaImpl.h:85
Definition: CdvaImpl.h:36
virtual unsigned char * getLocalBuffer()
Definition: CdvaImpl.h:132
unsigned char * localBuf
buffer for local descriptors
Definition: CdvaImpl.h:82
virtual void setGlobalUncompressedBufSz(unsigned long sz)
Definition: CdvaImpl.h:143
virtual void clear()
clear the shot container
unsigned long globalUncompressedBufSz
size of unencoded global buffer
Definition: CdvaImpl.h:87
static int nBitsSet(unsigned char byte)
Definition: CdvaImpl.h:228
double optMatch_tau
factor to define threshold to continue matching (opt matching modes 1 and 2)
Definition: CdvaImpl.h:188
void setEndTimeMs(unsigned long position_ms)
Set the start time of the frame in milliseconds.
virtual size_t write(std::ofstream &fout) const
write (appending) this segment to a file
virtual size_t read(std::ifstream &fin, mpeg7cdvs::CdvsServer *cdvsserver)
read (from the current position) this segment from a file
unsigned long globalBufSz
size of encoded global buffer
Definition: CdvaImpl.h:84
int skip_after
number of video frames to skip after decoding one
Definition: CdvaImpl.h:175
unsigned int getParity() const
Get the shot parity.
bool verboseMode
verbose mode indicator
Definition: CdvaImpl.h:171
unsigned long endTime
Definition: CdvaImpl.h:48
static const int SHOT_HEADER_SIZE
Definition: CdvaImpl.h:51
std::vector< mpeg7cdvs::CdvsDescriptor > keyframes
keyframes belonging to this shot
Definition: CdvaImpl.h:59
virtual void setHistoBufSz(unsigned long sz)
Definition: CdvaImpl.h:140
mpeg7cdvs::CdvsServer * cdvsserver
Definition: CdvaImpl.h:169
double drop_frame_th
drop frame threshold
Definition: CdvaImpl.h:176
unsigned long getStartTimeMs() const
Get the shot start time in milliseconds.
Extension of segment descriptor container to also hold compressed data for a segment.
Definition: CdvaImpl.h:78
static size_t addSegmentToDB(std::ifstream &fin, mpeg7cdvs::CdvsServer *cdvsserver, const std::string &relativepathname, bool optMatch)
read (from the current position) this segment from a file and store it into the CdvsServer DB ...
Definition: RefFeature.h:22
bool calc_desc_sizes
calculate size of descriptor components
Definition: CdvaImpl.h:181
Definition: CdvaImpl.h:36
Definition: CdvaImpl.h:36
std::vector< SegmentDescriptor > ShotDescriptorList
vector of shots descriptors of a video
Definition: CdvaImpl.h:152
bool optMatch
approx. matching of medoid-based descriptors
Definition: CdvaImpl.h:186
unsigned long startTime
Definition: CdvaImpl.h:47
size_t max_retrieved
maximum number of retrieved images
Definition: CdvaImpl.h:180
double optMatch_b
divider for fraction of key frames to check to decide for match (opt matching mode 2) ...
Definition: CdvaImpl.h:189
unsigned char * globalBuf
buffer for differential global descriptors
Definition: CdvaImpl.h:81
A class containing the results of an extraction operation.
Definition: cdva.h:38
A container for CdvsDescriptor instances belonging to the same video segment.
Definition: CdvaImpl.h:43
int cdvsMode
the CDVS mode that will be used to encode keyframe descriptors
Definition: CdvaImpl.h:173
virtual unsigned long getLocalBufSz()
Definition: CdvaImpl.h:136