Remoto - VFS: rutils.cpp Source File
Remoto - VFS
rutils.cpp
Go to the documentation of this file.
1 
2 #include <stdio.h>
3 
4 #include <QStringList>
5 #include <QJsonDocument>
6 #include <QFile>
7 #include <QRegularExpression>
8 
9 #include "VFS.h"
10 #include "rutils.h"
11 
32 QString rutils::cleanPath(QString path)
33 {
34  QStringList parts = path.split('/',Qt::SkipEmptyParts);
35  return parts.join('/');
36 }
37 
53 QByteArray rutils::resourceContents(QString resource, bool *ok, bool squashHash)
54 {
55  if (resource.at(0) != ':') //it must represent a resource path, not an arbtrary filesystem path
56  {
57  VFS::WARN( "rutils::resourceContents() can only access qrc resources." );
58  if (ok) *ok = false;
59  return "";
60  }
61 
62  QFile r(resource);
63  if (r.open(QIODevice::ReadOnly))
64  {
65  if (ok) *ok = true;
66 
67  if (squashHash)
68  return r.readAll().replace('#',"%23");
69  else
70  return r.readAll();
71  }
72  else
73  VFS::WARN( QString("Bad resourceContents path: %1").arg(resource) );
74 
75  if (ok) *ok = false;
76  return "";
77 }
78 
79 
90 QJsonObject rutils::jsonResource(QString resource, bool *ok)
91 {
92  bool rok;
93 
94  QByteArray r = rutils::resourceContents(resource, &rok);
95 
96  if (rok)
97  {
98  QJsonParseError err;
99  QJsonDocument d = QJsonDocument::fromJson( r, &err );
100 
101  if (err.error == QJsonParseError::NoError)
102  {
103  if (ok) *ok = true;
104  return d.object();
105  }
106  else
107  VFS::ERROR( resource+": "+err.errorString() );
108  }
109 
110  if (ok) *ok = false;
111  return QJsonObject();
112 }
113 
114 
123 //this list may ultimately want to come from some config entry, but for now it's hardcoded in
124 QStringList rutils::sequenceTypes = QStringList() << "jpe?g" << "tif?f" << "exr" << "png" << "dpx";
125 
132 QJsonValue rutils::parseSequenceSyntax(QString p)
133 {
134  QRegularExpression seqrx( "^(.*)\\.?\\[([-,x\\d]+)#(\\d+)\\]\\.(\\w{2,5})$"); //support extensions from 2-5 alphanumeric characters
135  QRegularExpressionMatch match = seqrx.match(p);
136 
137  if (match.hasMatch())
138  {
139  QJsonObject o;
140 
141  o["prefix"] = match.captured(1);
142  o["range"] = match.captured(2);
143  o["padding"] = match.captured(3).toInt();
144  o["extension"] = match.captured(4);
145 
146  o["entries"] = QJsonArray::fromStringList(expandSequenceListing(p));
147 
148  return o;
149  }
150 
151  return QJsonValue::Null;
152 }
153 
183 QJsonObject rutils::sequenceListing(QJsonObject l, QStringList types)
184 {
185  //1. convert the list of filenames to regexp signatures
186  //2. remove any signatures that only match one entry
187  //3. for every file in the list, find a matching signature that has the highest count and score(?) and add it to signatureFrames
188  // --any file that doesn't match any signature just gets added to the output list
189  //4. for every entry in signatureFrames, create a range-notated string and add it to the output list
190  //5. (the result list is already sorted by QJsonObject before returning)
191 
192  //printf("==SEQUENCE LISTING START==\n");
193 
194  QString name,nameEsc,t,t2;
195  bool container;
196  int o,c,d,capLen;
197 
198  QJsonObject r;
199  QRegExp digrx( "(-?\\d+)"); //an optional sign character followed by one or more digits
200  QString digrxs("(-?\\d{%1})"); //a string that will be inserted to make a new regex for each permutation of name/digits
201  QRegExp digrxrx( QRegExp::escape(digrxs.arg('#')).replace('#',"(-?\\d+)") ); //a capturing regex matching the regex used to match the signature, so we can get the number of digits
202  QRegExp typerx( "^.*\\.?("+types.join('|')+")$" ); //the list of sequenceTypes joined as a master regex
203  typerx.setCaseSensitivity(Qt::CaseInsensitive);
204 
205  //$, (,), *, +, ., ?, [, ,], ^, {, | and } //http://doc.qt.io/qt-4.8/qregexp.html#escape
206  QRegExp unescaperx( "\\\\(\\$|\\(|\\)|\\*|\\+|\\.|\\?|\\[|\\]|\\^|\\{|\\||\\})" );
207 
208  QHash<QRegExp,int> signatures;
209  QHash<QRegExp, QList<int> > signatureFrames;
210 
211 
212  //1. convert the list of filenames to regexp signatures
213  //========================================================
214  for( QJsonObject::const_iterator it = l.begin(); it != l.end(); ++it)
215  {
216  name = it.key();
217  container = it.value().toBool();
218 
219  if (!container && typerx.exactMatch(name)) //for files only, and only those that match the types we care about
220  {
221  nameEsc = QRegExp::escape(name); //a filename may contain regex characters... escape those
222 
223  for (c=d=o=0; c<100; c++)
224  {
225  t = nameEsc; //make a copy of the escaped name...
226  t2 = nameEsc; //make a copy of the escaped name...
227  o = digrx.indexIn(t,o);
228 
229  if (o>=0)
230  {
231  capLen = digrx.cap(1).length();
232  t.replace(o,capLen,digrxs.arg(capLen));
233  t2.replace(o,capLen,digrx.pattern());
234 
235  d++;
236  o+=capLen;
237 
238  signatures[QRegExp(t)]++; //increment the count for that signature
239  signatures[QRegExp(t2)]++; //increment the count for that signature
240  }
241  else
242  break;
243  }
244 
245  if (c == 100)
246  { VFS::ERROR("Too many loops when trying to do sequence listing for '"+nameEsc+"'");
247  return l;
248  }
249 
250  if (d==0) //it didn't contain any digit sequences, so just add it to the output list
251  r[name] = container;
252  }
253  else //for directories
254  r[name] = container;
255  }
256 
257 
258  //2. remove any signatures that only match one entry
259  //========================================================
260  QHash<QRegExp, int>::iterator pit = signatures.begin();
261  while (pit != signatures.end())
262  {
263  if (pit.value() == 1) pit = signatures.erase(pit); //remove any signatures that only have a count of one. Those can't be signatures.
264  else ++pit;
265  }
266 
267  //for ( QHash<QRegExp,int>::const_iterator pit = signatures.begin(); pit != signatures.end(); ++pit )
268  // printf("%s -> %d\n",qUtf8Printable(pit.key().pattern()),pit.value());
269 
270 
271  //3. for every file in the list, find a matching signature that has the highest count/score, and add the frame number to the signatureFrames list
272  //========================================================
273  QRegExp bestSignature,s;
274  int bestSignatureCount,frame;
275  for( QJsonObject::const_iterator it = l.begin(); it != l.end(); ++it)
276  {
277  name = it.key();
278  container = it.value().toBool();
279 
280  if (!container && typerx.exactMatch(name))
281  {
282  bestSignatureCount = 0;
283  for ( QHash<QRegExp,int>::const_iterator pit = signatures.begin(); pit != signatures.end(); ++pit )
284  {
285  s = pit.key();
286  c = pit.value();
287 
288  if (s.exactMatch(name))
289  {
290  if (c == bestSignatureCount) //sometimes two signatures match equally... take the shorter one
291  if (s.pattern().length() < bestSignature.pattern().length())
292  { bestSignature = s;
293  frame = s.cap(1).toInt();
294  }
295 
296  if (c > bestSignatureCount)
297  { bestSignatureCount = c;
298  bestSignature = s;
299  frame = s.cap(1).toInt();
300  }
301  }
302  }
303 
304  if (bestSignatureCount == 0)
305  r[name] = container;
306  else
307  { signatureFrames[bestSignature].append(frame);
308  //printf("%s -> %d\n",qUtf8Printable(bestSignature.pattern()),frame);
309  }
310  }
311  else //this should have already happened above
312  r[name] = container;
313  }
314 
315  //for ( QHash<QRegExp,QList<int> >::const_iterator fit = signatureFrames.begin(); fit != signatureFrames.end(); ++fit )
316  //{
317  // QRegExp sr = fit.key();
318  // QList<int> fl = fit.value();
319 
320  // printf("%s => ",qUtf8Printable(sr.pattern()));
321  // for (int i=0;i<fl.length();i++)
322  // printf("%d, ",fl.at(i));
323  // printf("\n");
324  //}
325 
326 
327  //4. for every entry in signatureFrames, create a range-notated string and add it to the output list
328  // range notation will be a comma-separated list of singletons and/or ranges:
329  // 1-100,245,500-1000,1963,2000-2200
330  //========================================================
331  for ( QHash<QRegExp,QList<int> >::const_iterator fit = signatureFrames.begin(); fit != signatureFrames.end(); ++fit )
332  {
333  QString sr = fit.key().pattern();
334  QList<int> fl = fit.value();
335  std::sort(fl.begin(),fl.end()); //sort the incoming frame list
336 
337  //printf("%s => ",qUtf8Printable(sr));
338  //for (int i=0;i<fl.length();i++)
339  // printf("%4d, ",fl.at(i));
340  //printf("\n");
341 
342  QList<int> deltas;
343  deltas << 0; //enter a dummy value to offset the pyramid
344  for (int i=0;i<fl.length()-1;i++)
345  deltas.append(fl[i+1]-fl[i]); //find the deltas/steps between each frame
346 
347  //printf("%s => ",qUtf8Printable(sr));
348  //for (int i=0;i<deltas.length();i++)
349  // printf("%4d, ",deltas.at(i));
350  //printf("\n");
351 
352  QStringList chunks;
353  int delta = 0; //stride between
354  int b,e,c; //begin,end
355  for (int i=1;i<=deltas.length();i++)
356  {
357  b = fl[i-1]; //grab the begin
358  c = 0;
359  delta = deltas[i];
360  while(i<deltas.length() && deltas[i] == delta) i++,c++; //increment to the next moment
361  e = fl[i-1]; //grab the end
362 
363  if (b==e) chunks << QString("%1").arg(b);
364  else if (c==1) chunks << QString("%1,%2").arg(b).arg(e);
365  else if (delta > 1) chunks << QString("%1-%2x%3").arg(b).arg(e).arg(delta);
366  else chunks << QString("%1-%2").arg(b).arg(e);
367  }
368 
369  //printf("%s => %s\n",qUtf8Printable(sr),qUtf8Printable(chunks.join(',')));
370 
371  int p = digrxrx.indexIn(sr); //position of regex in the regex
372  if (p > -1)
373  {
374  //printf("CAP: %d %s %s\n",p,qUtf8Printable( digrxrx.cap(0) ),qUtf8Printable( digrxrx.cap(1) ));
375 
376  int d = digrxrx.cap(1).toInt(); //number of digits in the regex
377  sr.replace( digrxrx, QString("[%1#%2]").arg(chunks.join(',')).arg(d) );
378  sr.replace( unescaperx, "\\1" );
379  //printf("%s\n",qUtf8Printable(sr));
380  r[sr] = false; //finally add the entry to the return list
381  }
382  else if (sr.indexOf(digrx.pattern()) > -1)
383  {
384  sr.replace( digrx.pattern(), QString("[%1@]").arg(chunks.join(',')) );
385  sr.replace( unescaperx, "\\1" );
386  //printf("%s\n",qUtf8Printable(sr));
387  r[sr] = false; //finally add the entry to the return list
388  }
389  else
390  { VFS::ERROR( "Unexplained regex error... reverting to non-sequence view: "+sr );
391  return l;
392  }
393  }
394 
395  //printf("==SEQUENCE LISTING END==\n");
396 
397  //return l;
398  return r;
399 }
400 
407 QStringList rutils::expandSequence(QString sequence)
408 {
409  if (sequence == "")
410  return QStringList();
411 
412  QRegularExpression seqrx( "^([-,x\\d]+)([@#]?)(\\d*)$");
413 
414  QRegularExpressionMatch match = seqrx.match(sequence);
415  if (match.hasMatch())
416  {
417  bool ok=false;
418  QStringList l;
419 
420  QString seq = match.captured(1);
421  QString hash = match.captured(2);
422  int pad = match.captured(3).toInt(&ok);
423 
424  if (!ok) pad = 0;
425  if (hash=="" || hash=="@") pad = 0;
426 
427  //printf( "match: %s\n", qUtf8Printable( QString("%1 %2 %3 %4").arg(prefix).arg(seq).arg(pad).arg(ext) ) );
428 
429  QStringList n = seq.split(",",Qt::SkipEmptyParts);
430 
431  QRegularExpression digrx( "^(-?\\d+)$"); //an optional sign character followed by one or more digits
432  QRegularExpression rangerx( "^(-?\\d+)-(-?\\d+)$"); //a pair of digrx connected by a '-' to create a range
433  QRegularExpression steprx( "^(-?\\d+)-(-?\\d+)x(\\d+)$"); //a rangerx with a step
434  QRegularExpressionMatch m;
435  int b=0,e=0,s=1;
436  ok = false;
437 
438  while (n.length())
439  {
440  QString c = n.takeFirst();
441  ok = false;
442 
443  if (!ok)
444  {
445  m = steprx.match(c);
446  if (m.hasMatch())
447  { b = m.captured(1).toInt();
448  e = m.captured(2).toInt();
449  s = m.captured(3).toInt();
450  ok = true;
451  }
452  }
453 
454  if (!ok)
455  {
456  m = rangerx.match(c);
457  if (m.hasMatch())
458  { b = m.captured(1).toInt();
459  e = m.captured(2).toInt();
460  s = 1;
461  ok = true;
462  }
463  }
464 
465  if (!ok)
466  {
467  m = digrx.match(c);
468  if (m.hasMatch())
469  { b = e = m.captured(1).toInt();
470  s = 1;
471  ok = true;
472  }
473  }
474 
475  if (!ok || b > e)
476  { VFS::ERROR("Invalid range: "+sequence);
477  return QStringList();
478  }
479 
480  for (int i=b;i<=e;i+=s)
481  //l << QString("%1.%2%3.%4").arg(prefix).arg(i<0?"-":"").arg(abs(i),pad.toInt(),10,QChar('0')).arg(ext);
482  l << QString("%1").arg(i,pad,10,QChar('0'));
483  }
484 
485  return l;
486  }
487  else
488  VFS::ERROR("Invalid sequence: "+sequence);
489 
490  return QStringList();
491 }
492 
499 QStringList rutils::expandSequenceListing(QString sequence)
500 {
501  QRegularExpression seqrx( "^(.*)\\.\\[([-,x\\d]+)#(\\d+)\\]\\.(\\w{2,5})$"); //support extensions from 2-5 alphanumeric characters
502 
503  QRegularExpressionMatch match = seqrx.match(sequence);
504  if (match.hasMatch())
505  {
506  QStringList l;
507 
508  QString prefix = match.captured(1);
509  QString seq = match.captured(2);
510  QString pad = match.captured(3);
511  QString ext = match.captured(4);
512 
513  //printf( "match: %s\n", qUtf8Printable( QString("%1 %2 %3 %4").arg(prefix).arg(seq).arg(pad).arg(ext) ) );
514 
515  QStringList n = seq.split(",",Qt::SkipEmptyParts);
516 
517  QRegularExpression digrx( "^(-?\\d+)$"); //an optional sign character followed by one or more digits
518  QRegularExpression rangerx( "^(-?\\d+)-(-?\\d+)$"); //a pair of digrx connected by a '-' to create a range
519  QRegularExpression steprx( "^(-?\\d+)-(-?\\d+)x(\\d+)$"); //a rangerx with a step
520  QRegularExpressionMatch m;
521  int b=0,e=0,s=1;
522  bool ok = false;
523 
524  while (n.length())
525  {
526  QString c = n.takeFirst();
527  ok = false;
528 
529  if (!ok)
530  {
531  m = steprx.match(c);
532  if (m.hasMatch())
533  { b = m.captured(1).toInt();
534  e = m.captured(2).toInt();
535  s = m.captured(3).toInt();
536  ok = true;
537  }
538  }
539 
540  if (!ok)
541  {
542  m = rangerx.match(c);
543  if (m.hasMatch())
544  { b = m.captured(1).toInt();
545  e = m.captured(2).toInt();
546  s = 1;
547  ok = true;
548  }
549  }
550 
551  if (!ok)
552  {
553  m = digrx.match(c);
554  if (m.hasMatch())
555  { b = e = m.captured(1).toInt();
556  s = 1;
557  ok = true;
558  }
559  }
560 
561  if (!ok || b > e)
562  { VFS::ERROR("Invalid range: "+sequence);
563  return QStringList();
564  }
565 
566  for (int i=b;i<=e;i+=s)
567  //l << QString("%1.%2%3.%4").arg(prefix).arg(i<0?"-":"").arg(abs(i),pad.toInt(),10,QChar('0')).arg(ext);
568  l << QString("%1.%2.%3").arg(prefix).arg(i,pad.toInt(),10,QChar('0')).arg(ext);
569  }
570 
571  //printf("LIST: %s\n",qPrintable(l.join(" ")));
572 
573  return l;
574  }
575  else
576  VFS::ERROR("Invalid sequence: "+sequence);
577 
578  return QStringList();
579 }
580 
581 
582 
592 QByteArray rutils::encodeBase64Path( QJsonObject p )
593 {
594  QJsonDocument d(p);
595  QByteArray r = d.toJson().toBase64();
596  QByteArray b = r.replace('/','_');
597 
598  return b;
599 }
600 
601 
602 
613 QJsonObject rutils::decodeBase64Path( QByteArray p, bool *ok )
614 {
615  QByteArray r = p.replace('_','/');
616  QByteArray b = QByteArray::fromBase64(r);
617 
618  QJsonParseError e;
619  QJsonDocument d = QJsonDocument::fromJson(b,&e);
620 
621  if (!d.isNull() && e.error == QJsonParseError::NoError)
622  { *ok = true;
623  return d.object();
624  }
625 
626  *ok = false;
627  return QJsonObject();
628 }
629 
640 QString rutils::unescapeXMLAttribute( QString attribute )
641 {
642  attribute = attribute.replace("&lt;","<");
643  attribute = attribute.replace("&gt;",">");
644 
645  return attribute;
646 }
647 
648 
657 //https://www.tutorialspoint.com/How-to-execute-a-command-and-get-the-output-of-command-within-Cplusplus-using-POSIX
658 QStringList rutils::exec(QString command,bool *ok)
659 {
660  QString result;
661 
662  // Open pipe to file
663  FILE* pipe = popen(qUtf8Printable(command), "r");
664  if (!pipe)
665  {
666  if (ok) *ok = false;
667  return QStringList();
668  }
669 
670  // read till end of process:
671  char buffer[128];
672  while (!feof(pipe))
673  {
674  // use buffer to read and add to result
675  if (fgets(buffer, 128, pipe) != nullptr)
676  result += buffer;
677  }
678 
679  pclose(pipe);
680 
681  if (ok) *ok = true;
682  return result.split("\n",Qt::SkipEmptyParts);
683 }
static void ERROR(QString message, int level=0, QString user="server")
Send a message to the VFS::_errors VFS_stream.
Definition: VFS.cpp:307
static void WARN(QString message, int level=0, QString user="server")
Send a message to the VFS::_warnings VFS_stream.
Definition: VFS.cpp:258
setter name
a setter DOCME
getter path
a getter DOCME
QStringList expandSequence(QString sequence)
Expand a sequence-notated list into a list of numbers.
Definition: rutils.cpp:407
QString cleanPath(QString path)
Clean and normalize a VFS path.
Definition: rutils.cpp:32
QStringList expandSequenceListing(QString sequence)
Expand a sequence-notated string into a list of names.
Definition: rutils.cpp:499
QStringList sequenceTypes
A list of regular expressions which can represent image or file sequences.
Definition: rutils.cpp:124
QJsonObject decodeBase64Path(QByteArray p, bool *ok)
Decode a base64 string as a JSON object.
Definition: rutils.cpp:613
QStringList exec(QString command, bool *ok=nullptr)
execute a command in a shell, and return the resulting output as a QStringList
Definition: rutils.cpp:658
QJsonObject jsonResource(QString resource, bool *ok=nullptr)
Fetch the contents of a Qt resource as a QJsonObject.
Definition: rutils.cpp:90
QJsonValue parseSequenceSyntax(QString path)
Check if a path or string matches our syntax for file sequences.
Definition: rutils.cpp:132
QJsonObject sequenceListing(QJsonObject l, QStringList types=sequenceTypes)
Given a list of filenames and a regex list, collapse the listing to sequences when possible.
Definition: rutils.cpp:183
QString unescapeXMLAttribute(QString attribute)
Un-escape an XML attribute.
Definition: rutils.cpp:640
QByteArray encodeBase64Path(QJsonObject p)
Encode a JSON object as a base64 string.
Definition: rutils.cpp:592
QByteArray resourceContents(QString resource, bool *ok=nullptr, bool squashHash=false)
Fetch the contents of a Qt resource file.
Definition: rutils.cpp:53