Convert HTML Table into EXCEL , MS Word , PDF in Java

Report generation in different formats is troublesome in most of the projects. However, there still is an easier way to do for experienced engineers. Here I will recommend a jar which can directly convert any HTML Table with CSS into 3 Main report formats (EXCEL, WORD, PDF). This project saves developers tons of efforts to manually write a utility class to parse the XPath in Apache POI or Doc4j and generate reports. here is the address :

https://github.com/alanhay/html-exporter

Here is the code example :

public static void main(String[] args) throws FileNotFoundException, IOException {
  
 String html = IOUtils.toString(new FileInputStream("./Sample.html"));
  
 new ExcelExporter().exportHtml(html, new File("./report.xlsx"));
}

If you are using this jar, 3 points need to be noted :

1. CSS Style must be defined in the <Style> tag.

<style type="text/css">
.pNewCounter {
  background: #FFC000;
 }

table, tbody, tfoot, thead, tr, th, td, footer, header {
  margin: 0;
  padding: 0;
  border: 0;
  font-size: 100%;
  font: inherit;
  vertical-align: baseline;
 }
</style>

                    2. Date format data must have data-date-cell-format attribute                              

   <tr>
          <td>Creation Date</td>
   <td data-date-cell-format="dd/MM/yyyy">06/08/2018</td>
   </tr>

3. Text Data must have data-text-cell attribute

    <tr>
   <td>Offer Expiry Date</td>
   <td data-date-cell-format="dd/MM/yyyy">14/09/2018</td> 
  <td data-date-cell-format="dd/MM/yyyy">14/09/2018</td>
    </tr>
 
 
    <tr>
   <td>Offered Amount</td>
   <td data-text-cell='true' >3,500,000.00</td>
                <td data-text-cell='true' ><strong>3,600,000.00</strong></td>
   </tr>
 
 
    <tr>
   <td>Offered Amount (%) </td>
   <td data-text-cell='true' >35.000%</td>
                <td data-text-cell='true'><strong>36.000%</strong></td>
    </tr>

Sample HTML :

https://github.com/callow/html-exporter/blob/master/Sample.html

Converted Result :

https://github.com/callow/html-exporter/blob/master/report.xlsx

Credit Card Recognition

Credit Card recognition will be based on the modal from OpenCV,  It's not necessary to train our own modal based on data as the card number format are mostly all the same.

1.  download the OpenCV jar file and add into classpath or import as maven, then static import the OpenCV c++ library as below:

static {
        System.loadLibrary(Core.NATIVE_LIBRARY_NAME);
    }

2. read the credit card image into the program into the Mat object

public static void main(String[] args) {
        Mat srcImage = loadImage("img/card.jpg");
}

public static  Mat loadImage(String path) {
 Mat newImage = Imgcodecs.imread(path);
 return newImage;
}

3. turn the card image into grey color (grey scale process)

public static void main(String[] args) {
 Mat srcImage = loadImage("img/card.jpg");
 Mat grey = grey(srcImage);
}

public static Mat grey(Mat srcMat) {
 Mat dest = new Mat (); 
 Imgproc.cvtColor(srcMat, dest, Imgproc.COLOR_RGB2GRAY);
 return dest;
}

4. turn the grey scale image into a black & white image. (binary process)

public static void main(String[] args) 
{
 Mat srcImage = loadImage("img/card.jpg");
 Mat grey = grey(srcImage);
 Mat binary = blackWhite(grey);
}

public static Mat blackWhite(Mat grayMat) {
  Mat binaryMat = new Mat(grayMat.height(),grayMat.width(),CvType.CV_8UC1);
         Imgproc.threshold(grayMat, binaryMat, 30, 200, Imgproc.THRESH_BINARY);
         return binaryMat;
}

5. Erode the black & white image to make the word bolder and clearer so as to be easy to be recognized.

public static void main(String[] args) {
 Mat srcImage = loadImage("img/card.jpg"); 
 Mat grey = grey(srcImage);
        Mat binary = blackWhite(grey);
 Mat erode = imageErode(binary);
}
public static Mat imageErode(Mat srcMat) {
 Mat destMat = new Mat();
 Mat element = Imgproc.getStructuringElement(Imgproc.MORPH_RECT, new Size(3, 3));
        Imgproc.erode(srcMat,destMat,element);
        return destMat;
}

6. crop the key region to remove the noise.

public static void main(String[] args) {
 Mat srcImage = loadImage("img/card.jpg");
 Mat grey = grey(srcImage);
        Mat binary = blackWhite(grey);
 Mat erode = imageErode(binary);
        Mat adjust = filterAndcut(erode);
}
public static Mat  filterAndcut(Mat destMat) {
  int a =0, b=0, state = 0;
         for (int y = 0; y < destMat.height(); y++) //row
         {
             int count = 0;
             for (int x = 0; x < destMat.width(); x++) //column
             {
                 // get the row pixel value 
                 byte[] data = new byte[1];
                 destMat.get(y, x, data);
                 if (data[0] == 0)
                     count = count + 1;
             }
             if (state == 0)//not a valid row
             {
                 if (count >= 150)//find a valid row
                 {//valid row qualify a 10 pixel noise
                     a = y;
                     state = 1;
                 }
             }
             else if (state == 1)
             {
                 if (count <= 150)// find a valid row
                 {//valid row qualify a 10 pixel noise
                     b = y;
                     state = 2;
                 }
             }
         }
         System.out.println("filter upper bound "+Integer.toString(a));
         System.out.println("filter lower bound "+Integer.toString(b));


         //crop the valid region
         Rect rect = new Rect(0,a,destMat.width(),b - a);
         Mat resMat = new Mat(destMat,rect);
         return resMat;
 }

7. recognize and print the result.

public static void main(String[] args) {
  Mat srcImage = loadImage("img/card.jpg");
  Mat grey = grey(srcImage); 
  Mat binary = blackWhite(grey);
  Mat erode = imageErode(binary);
  Mat adjust = filterAndcut(erode);
  
  printResult(matToBufferImage(adjust)); 
}

public static BufferedImage matToBufferImage(Mat grayMat) {
 if (grayMat == null) {
      return null;
 }
 byte[] data1 = new byte[grayMat.rows() * grayMat.cols() * (int)(grayMat.elemSize())];
 grayMat.get(0, 0, data1);
 BufferedImage image1 = new BufferedImage(grayMat.cols(), grayMat.rows(),BufferedImage.TYPE_BYTE_GRAY);                           
 image1.getRaster().setDataElements(0, 0, grayMat.cols(), grayMat.rows(), data1);
 return image1;
}

public static void printResult(BufferedImage src) {
 ITesseract instance = new Tesseract();
 instance.setDatapath("F:/Tess4J-3.4.8-src/Tess4J/tessdata"); // Credit card trained data
 long startTime = System.currentTimeMillis();
 String ocrResult;
 try {
       ocrResult = instance.doOCR(src);
       System.out.println("OCR Result: \n" + ocrResult + "\n spend：" + (System.currentTimeMillis() - startTime) + "ms");
 } catch (TesseractException e) {
       e.printStackTrace();
 } 
}

That's all for this project.

JavaScript !function()

If you like to read some JavaScript plugin source code online you may frequently encounter something like this below :

so how do we interpret this?

If we print this out it is like this below, why? because an anonymous function is undefined, and !undefined is true, we don't care whether it returns true or false but care that the JavaScript engine can parse it perfectly. 

!function(){alert('sd')}()        // true

But you may wonder how does an anonymous function can call itself with no error.

We normally see the below formats more frequently :

(function(){alert('123')})()        // true
(function(){alert('123')}())        // true

Anyway whichever format it is, its main purpose to put a bracket around is to turn a function declaration into an Expression which can be parsed/ Run. for example

function a(){alert('zsd')}   // undefined

This is a function declaration if we put a bracket behind directly, JavaScript Engine certainly doesn't understand. so Syntax error will come.

function a(){alert('iifksp')}()  // SyntaxError: unexpected_token

This code makes the engine confused between function declaration and function call. However, putting a bracket around is different, it turns a function declaration into an expression, the engine will understand it is no more a function declaration but an expression, So this expression will be called when the code is run into this line.

In short, Any grammar that can help to make the engine clearly differentiate function declaration and expression will be accepted by Javascript. For example:

var i = function(){return 10}();        // undefined
1 && function(){return true}();        // true
1, function(){alert('as')}();        // undefined

Assignment Operators,  Logic Operators, and even a comma can tell then engine that this is not a function declaration but a function expression. The unary operator is said to be the fastest way to clear the ambiguity for the engine, exclamation mark (!) is just one of them. here come some examples:

!function(){alert('as')}()        // true
+function(){alert('as')}()        // NaN
-function(){alert('as')}()        // NaN
~function(){alert('as')}()        // -1

void function(){alert('as')}()        // undefined
new function(){alert('as')}()        // Object
delete function(){alert('as')}()        // true

(function(){alert('as')})()        // undefined
(function(){alert('as')}())        // undefined

More Examples  

Option	Code
!	!function(){;}()
+	+function(){;}()
-	-function(){;}()
~	~function(){;}()
-1	(function(){;})()
-2	(function(){;}())
void	void function(){;}()
new	new function(){;}()
delete	delete function(){;}()
=	var i = function(){;}()
&&	1 && function(){;}()
||	0 || function(){;}()
&	1 & function(){;}()
|	1 | function(){;}()
^	1 ^ function(){;}()
,	1, function(){;}()

Java 8 - Parallel data processing

In Java 7 , If you want to process data in parallel, you may consider the ForkJoinPool which is one of the threadpool factory with number of initial thread equalising the number of procesor of the computer :

Runtime.getRuntime().available-Processors() // show initial number of thread

System.setProperty("java.util.concurrent.ForkJoinPool.common.parallelism","12"); // change the number of initial thread

In current multi-core PC world, parallel processing will  mostly increase the performence. And the way to implement the parallelism is as simple as just adding one methond parallel()  into the normal Stream. This parallel() will automaticly make use of the ForkJoinPool behind and dynamicly assign the data into threads.

public static long parallelSum(long n) {
     return Stream.iterate(1L, i -> i + 1).limit(n).parallel().reduce(0L, Long::sum);
}

On the contrary , if you want to abandon parallel(), a sequential() will help you switch back smoothly. Well , In practice, whether to choose sequential() , parallel() or normal for loop , all depend on practical measurement, There is no one golden rule saying that parallel() is the best, you must measure the time and choose which one to apply.

long start = System.nanoTime();
// apply batch calculation here
long duration = (System.nanoTime() - start) / 1_000_000;

In most of the cases , the efficiency lost in the parallel processing is due to autobox problems,  one of the methods to increse the paralle() is to avoid the autobox using LongStream, IntStream... This is important!

public static long parallelRangedSum(long n) {
    return LongStream.rangeClosed(1, n).parallel().reduce(0L, Long::sum);
}

ForkJoinPool

Above, I have mentioned the ForkJoinPool in Java 7 and the parallel() in Java 8 is implemented by it . So how do people use this ForkJoinPool? Answer :  extends RecursiveTask<T>

public class ForkJoinSumCalculator extends RecursiveTask<Long> {
       // constructors and fields .... 
@Override
 protected Long compute() { // rewrite abstract method
    if (length <= 1000) {
  // Don't use parallel
    }
    ForkJoinSumCalculator leftTask = new ForkJoinSumCalculator(20000, 1, 10000);
  
    leftTask.fork(); // start left half tasks asyncronously in another ForkJoinPool
  
    ForkJoinSumCalculator rightTask = new ForkJoinSumCalculator(20000, 10001, 20000);
  
    Long rightResult = rightTask.compute(); // start the right half tasks
  
    Long leftResult = leftTask.join(); // read the left half tasks result. blocking
  
    return leftResult + rightResult; // combine
 }
}

Parallel() is not only implemented by ForkJoinPool but also with the help of Spliterator. With these 2 weapons, parallel() in Java 8 can flexibly split big task ,compute and join together to return values.

Spliterator

Spliterator looks similar to iterator in versions before Java 8. It also helps to iterate data but in parallel way.

public interface Spliterator<T> {
   boolean tryAdvance(Consumer<? super T> action); // iterate all elements Consumer provides
   Spliterator<T> trySplit(); // split data into another Spliterator
   long estimateSize(); // return the size of collection
   int characteristics();
}

Ok, I am still working to produce an example , I will give it out later.

Java 8 - Data collection

Data collection is simplified in Java 8 syntax very much. you only need to point out "the expected result " not to worry about how to collect. In Java 8 , there exist a factory class "Collectors", let't see how it applies.

Math Problems 

long howManyDishes = menu.stream().collect(Collectors.counting()); // ==
long howManyDishes2 = menu.stream().count();
  
Comparator<Dish> dishCaloriesComparator = Comparator.comparingInt(Dish::getCalories); 
Optional<Dish> mostCalorieDish = menu.stream()
      .collect(Collectors.maxBy(dishCaloriesComparator)); // get object with max calories
  
// get the total calories
int totalCalories = menu.stream().collect(Collectors.summingInt(Dish::getCalories));
  
// get the average calories
double avgCalories = menu.stream().collect(Collectors.averagingInt(Dish::getCalories));
  
// get total, average, max, min 
IntSummaryStatistics menuStatistics = menu.stream().collect(

                               Collectors.summarizingInt(Dish::getCalories));

String Concatenation

String shortMenu = menu.stream().map(Dish::getName).collect(Collectors.joining(", "));

Reduce

// get total calories in reduing way
int totalCalories2 = menu.stream().collect(Collectors.reducing(0, Dish::getCalories,

                                           (i, j) -> i + j));
  
// get the max calories
Optional<Dish> mostCalorieDish2 = menu.stream().collect(Collectors.reducing(                        (d1, d2) -> d1.getCalories() > d2.getCalories() ? d1 : d2));
  
// self-implement toList collector
Stream<Integer> stream = Arrays.asList(1, 2, 3, 4, 5, 6).stream();
List<Integer> numbers = stream.reduce(new ArrayList<Integer>(), // init value
  (List<Integer> l, Integer e) -> { l.add(e);return l; }, // process
  (List<Integer> l1, List<Integer> l2) -> {

                               l1.addAll(l2);return l1; }); // final process
  
// total calories
int totalCalories3 = menu.stream().collect(Collectors.reducing(0,Dish::getCalories,

                                                                 Integer::sum));
int totalCalories4 = menu.stream().mapToInt(Dish::getCalories).sum(); // old way

Group By

Group by is like SQL group by statement., you can do nexted grouping , criteria grouping ...

// group by type , type is an enum
Map<Dish.Type, List<Dish>> dishesByType = menu.stream().collect(

                                          Collectors.groupingBy(Dish::getType));
  
// criteria grouping
Map<CaloricLevel, List<Dish>> dishesByCaloricLevel = menu.stream().collect(
  Collectors.groupingBy(dish -> {
   if (dish.getCalories() <= 400)  {
    return CaloricLevel.DIET; 
   } else if (dish.getCalories() <= 700) {
    return CaloricLevel.NORMAL;   
   }else {
    return CaloricLevel.FAT;
   }}));
// nested grouping
Map<Dish.Type, Map<CaloricLevel, List<Dish>>> dishesByTypeCaloricLevel = 

          menu.stream().collect(
   Collectors.groupingBy(Dish::getType, // 1st grouping
    Collectors.groupingBy(dish -> { // 2ed grouping
     if (dish.getCalories() <= 400) 
      return CaloricLevel.DIET;
     else if (dish.getCalories() <= 700) 
      return CaloricLevel.NORMAL;
     else 
      return CaloricLevel.FAT;
     })));
  
// collecting data with grouping
// result is : {MEAT=3, FISH=2, OTHER=4}
Map<Dish.Type, Long> typesCount = menu.stream().collect(

                  Collectors.groupingBy(Dish::getType, Collectors.counting()));
  
// get the max calories of each group , group by type
//result is : {FISH=Optional[salmon], OTHER=Optional[pizza], MEAT=Optional[pork]}
Map<Dish.Type, Optional<Dish>> mostCaloricByType = menu.stream().collect(

                  Collectors.groupingBy(Dish::getType,
    Collectors.maxBy(Comparator.comparingInt(Dish::getCalories))));
  
// get the max colories of each group , retrive value of each optional from above
//result is {FISH=salmon, OTHER=pizza, MEAT=pork}
Map<Dish.Type, Dish> mostCaloricByType2 = menu.stream().collect(

                Collectors.groupingBy(Dish::getType, // group by type
  Collectors.collectingAndThen(Collectors.maxBy(

                Comparator.comparingInt(Dish::getCalories)), //transformed Collector

  Optional::get))); // function of transfering
  
// grouping by can pass in 2 params , the second param will apply to all elements of each   group.
Map<Dish.Type, Integer> totalCaloriesByType = menu.stream().collect(

    Collectors.groupingBy(Dish::getType, Collectors.summingInt(Dish::getCalories)));
  
  
// get all caloricLevels of each Dish Type
// Collectors.mapping is powerful like map()
Map<Dish.Type, Set<CaloricLevel>> caloricLevelsByType = menu.stream().collect(
 Collectors.groupingBy(Dish::getType, Collectors.mapping( dish -> { 
   if (dish.getCalories() <= 400) {
    return CaloricLevel.DIET; 
   } else if (dish.getCalories() <= 700) {
    return CaloricLevel.NORMAL;
   } else {
    return CaloricLevel.FAT;
   }},Collectors.toSet())));
  
// point out which kind of set you want to use
Map<Dish.Type, Set<CaloricLevel>> caloricLevelsByType2 = menu.stream().collect(
 Collectors.groupingBy(Dish::getType, Collectors.mapping( dish -> { 
  if (dish.getCalories() <= 400) {
   return CaloricLevel.DIET;
  } else if (dish.getCalories() <= 700) {
   return CaloricLevel.NORMAL;
  } else {
   return CaloricLevel.FAT;  
  }},Collectors.toCollection(HashSet::new))));

Partition By

Partion By is similar to Group by but the input parameter is limited to "Predicate".

//{false=[pork, beef, chicken, prawns, salmon],
//true=[french fries, rice, season fruit, pizza]}
Map<Boolean, List<Dish>> partitionedMenu =
 menu.stream().collect(Collectors.partitioningBy(Dish::isVegetarian));
  
List<Dish> vegetarianDishes = partitionedMenu.get(true); // to retrive values
// == 
List<Dish> vegetarianDishes2 = menu.stream().filter(Dish::isVegetarian).collect(

                                                    Collectors.toList());
    
// nexted partioning & grouping , producing a 2 level grouping
//{false={FISH=[prawns, salmon], MEAT=[pork, beef, chicken]},
// true={OTHER=[french fries, rice, season fruit, pizza]}}
Map<Boolean, Map<Dish.Type, List<Dish>>> vegetarianDishesByType = menu.stream().collect( Collectors.partitioningBy(Dish::isVegetarian,
      Collectors.groupingBy(Dish::getType))); 

                                  // getType is not predicate so use group by