Out of memory for very large file

[peter.morgan.homeiname.co]

Hello,

I'm using Jasper Reports v6.5.1 with Java8 to produce a CSV file. I'm finding when there are a very large number of objects to process I run out of memory. I found an article about the virtualizer (http://jasperreports.sourceforge.net/sample.reference/virtualizer/index.html) and I had high hopes this would help me. However it does not seem to improve things at all. As far as I can tell it is not writing to the virtualizer file at all so I'm wondering if I've missed something important in the setup. 

I've tried to reduce the problem down to a simple Jasper report file and test case. 

Any help or suggestions would be most gratefully recieved,

Kind regards 

Pete

---

Test case:

import java.io.File;import java.util.ArrayList;import java.util.List;import java.util.Map;import java.util.concurrent.ConcurrentHashMap;import org.junit.Test;import net.sf.jasperreports.engine.JRAbstractExporter;import net.sf.jasperreports.engine.JREmptyDataSource;import net.sf.jasperreports.engine.JRException;import net.sf.jasperreports.engine.JRExporterParameter;import net.sf.jasperreports.engine.JRParameter;import net.sf.jasperreports.engine.JasperCompileManager;import net.sf.jasperreports.engine.JasperFillManager;import net.sf.jasperreports.engine.JasperPrint;import net.sf.jasperreports.engine.JasperReport;import net.sf.jasperreports.engine.design.JasperDesign;import net.sf.jasperreports.engine.export.JRCsvExporter;import net.sf.jasperreports.engine.fill.JRFileVirtualizer;import net.sf.jasperreports.engine.xml.JRXmlLoader;public class JasperExportHelper_VeryVeryLargeCSV_IntegrationTest {    private static final int NUMBER_OF_RECORDS = 10000000;    @Test    public void testCallItemisationByChargeGroup() throws JRException {        String invoiceTemplateName = "./src/test/resources/test.jrxml";        // load the test jasperReport        File templateFile = new File(invoiceTemplateName);        JasperReport jasperReport = null;        if (templateFile != null) {            JasperDesign jasperDesign = JRXmlLoader.load(templateFile);            jasperReport = JasperCompileManager.compileReport(jasperDesign);        }        Map<String, Object> invoiceParametersMap = new ConcurrentHashMap<>();        List<TestDto> dtos = new ArrayList<>();        for (int i = 1; i < NUMBER_OF_RECORDS; i++) {            TestDto f1 = new TestDto("Bob" + i, "Jones", "SN148LT", i);            dtos.add(f1);        }        invoiceParametersMap.put("LIST_OF_CALL_SUMMARIES_BY_CHRGRP", dtos);        JRFileVirtualizer virtualizer = new JRFileVirtualizer(2,                "/Users/pete/work/si/code/elevate/billing/billing-document/target");        virtualizer.setReadOnly(false);        long start = System.currentTimeMillis();        invoiceParametersMap.put(JRParameter.REPORT_VIRTUALIZER, virtualizer);        JasperPrint jasperPrint = JasperFillManager.fillReport(jasperReport, invoiceParametersMap,                new JREmptyDataSource());        long end = System.currentTimeMillis();        System.out.println("Filling: " + ((end - start) / 1000) + " second(s)");        start = System.currentTimeMillis();        String path = "/Users/pete/work/si/code/elevate/billing/billing-document/target";        File invoiceDocumentPath = new File(path);        invoiceDocumentPath.mkdirs();        // export as PDF file and copy the contents to the output stream        File invoiceDocument = new File(path + "/test.csv");        @SuppressWarnings("rawtypes")        JRAbstractExporter exporter = new JRCsvExporter();        jasperPrint.setProperty("net.sf.jasperreports.export.csv.exclude.origin.band.csvSummary", "summary");        jasperPrint.setProperty("net.sf.jasperreports.export.csv.exclude.origin.band.1", "title");        jasperPrint.setProperty("net.sf.jasperreports.export.csv.exclude.origin.band.2", "pageFooter");        jasperPrint.setProperty("net.sf.jasperreports.export.csv.exclude.origin.keep.first.band.1", "columnHeader");        exporter.setParameter(JRExporterParameter.JASPER_PRINT, jasperPrint);        exporter.setParameter(JRExporterParameter.CHARACTER_ENCODING, "UTF-8");        exporter.setParameter(JRExporterParameter.OUTPUT_FILE, invoiceDocument);        exporter.exportReport();        end = System.currentTimeMillis();        System.out.println("Export: " + ((end - start) / 1000) + " second(s)");        virtualizer.cleanup();    }    public static class TestDto {        private String firstName;        private String lastName;        private String postcode;        private int count;        public TestDto(String firstName, String lastName, String postcode, int count) {            super();            this.firstName = firstName;            this.count = count;            this.lastName = lastName;            this.postcode = postcode;        }        public String getFirstName() {            return firstName;        }        public void setFirstName(String firstName) {            this.firstName = firstName;        }        public int getCount() {            return count;        }        public void setCount(int count) {            this.count = count;        }        public String getLastName() {            return lastName;        }        public void setLastName(String lastName) {            this.lastName = lastName;        }        public String getPostcode() {            return postcode;        }        public void setPostcode(String postcode) {            this.postcode = postcode;        }    }}

Jasper test.jrxml file:

<?xml version="1.0" encoding="UTF-8"?><!-- Created with Jaspersoft Studio version last--><jasperReport xmlns="http://jasperreports.sourceforge.net/jasperreports" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://jasperreports.sourceforge.net/jasperreports http://jasperreports.sourceforge.net/xsd/jasperreport.xsd" name="Call Summary By Charge Group" language="groovy" pageWidth="1600" pageHeight="842" columnWidth="1600" leftMargin="0" rightMargin="0" topMargin="0" bottomMargin="0" isIgnorePagination="true" uuid="902cc340-9149-4aff-b21a-5bc2215a2f08">    <property name="net.sf.jasperreports.export.xls.create.custom.palette" value="false" />    <property name="net.sf.jasperreports.export.xls.one.page.per.sheet" value="false" />    <property name="net.sf.jasperreports.export.xls.remove.empty.space.between.rows" value="false" />    <property name="net.sf.jasperreports.export.xls.remove.empty.space.between.columns" value="false" />    <property name="net.sf.jasperreports.export.xls.white.page.background" value="true" />    <property name="net.sf.jasperreports.export.xls.detect.cell.type" value="true" />    <property name="net.sf.jasperreports.export.xls.size.fix.enabled" value="false" />    <property name="net.sf.jasperreports.export.xls.ignore.graphics" value="false" />    <property name="net.sf.jasperreports.export.xls.collapse.row.span" value="false" />    <property name="net.sf.jasperreports.export.xls.ignore.cell.border" value="false" />    <property name="net.sf.jasperreports.export.xls.ignore.cell.background" value="false" />    <property name="net.sf.jasperreports.export.xls.max.rows.per.sheet" value="0" />    <property name="net.sf.jasperreports.export.xls.wrap.text" value="true" />    <property name="net.sf.jasperreports.export.xls.use.timezone" value="false" />    <subDataset name="Call Itemisation Rate ChargeGroup Records" uuid="c37bd583-e342-4c64-a222-581f32745903">        <field name="firstName" class="java.lang.String"/>        <field name="lastName" class="java.lang.String"/>        <field name="postcode" class="java.lang.String"/>        <field name="count" class="java.lang.Integer"/>    </subDataset>    <parameter name="LIST_OF_CALL_SUMMARIES_BY_CHRGRP" class="java.util.List"/>    <variable name="LIST_OF_CALL_SUMMARIES_BY_CHRGRP" class="java.util.List">        <variableExpression><![CDATA[$P{LIST_OF_CALL_SUMMARIES_BY_CHRGRP} != null ? $P{LIST_OF_CALL_SUMMARIES_BY_CHRGRP} : new java.util.ArrayList()]]></variableExpression>    </variable>    <title>        <band height="13" splitType="Stretch">            <staticText>                <reportElement x="0" y="0" width="100" height="13" uuid="7db72793-c591-4620-a60e-9af912a422ad"/>                <textElement>                    <font isBold="false"/>                </textElement>                <text><![CDATA[First Name]]></text>            </staticText>            <staticText>                <reportElement x="0" y="0" width="200" height="13" uuid="7db72793-c591-4620-a60e-9af912a422ad"/>                <textElement>                    <font isBold="false"/>                </textElement>                <text><![CDATA[Last Name]]></text>            </staticText>            <staticText>                <reportElement x="0" y="0" width="300" height="13" uuid="7db72793-c591-4620-a60e-9af912a422ad"/>                <textElement>                    <font isBold="false"/>                </textElement>                <text><![CDATA[Postcode]]></text>            </staticText>            <staticText>                <reportElement x="0" y="0" width="400" height="13" uuid="7db72793-c591-4620-a60e-9af912a422ad"/>                <textElement>                    <font isBold="false"/>                </textElement>                <text><![CDATA[Count]]></text>            </staticText>        </band>    </title>    <detail>        <band height="13" splitType="Stretch">            <printWhenExpression><![CDATA[$V{LIST_OF_CALL_SUMMARIES_BY_CHRGRP} != null && $V{LIST_OF_CALL_SUMMARIES_BY_CHRGRP}.size() > 0]]></printWhenExpression>            <componentElement>                <reportElement x="0" y="0" width="2000" height="13" uuid="c2406085-2c1c-4646-a9d4-babb1c0376cd"/>                <c:list xmlns:c="http://jasperreports.sourceforge.net/jasperreports/components" xsi:schemaLocation="http://jasperreports.sourceforge.net/jasperreports/components http://jasperreports.sourceforge.net/xsd/components.xsd" printOrder="Vertical">                    <datasetRun subDataset="Call Itemisation Rate ChargeGroup Records" uuid="8a775787-05a8-4f08-a831-dbf869aa1359">                        <parametersMapExpression><![CDATA[$P{REPORT_PARAMETERS_MAP}]]></parametersMapExpression>                        <dataSourceExpression><![CDATA[new net.sf.jasperreports.engine.data.JRBeanCollectionDataSource($V{LIST_OF_CALL_SUMMARIES_BY_CHRGRP})]]></dataSourceExpression>                    </datasetRun>                    <c:listContents height="13">                        <textField isBlankWhenNull="true">                            <reportElement x="0" y="0" width="100" height="13" uuid="36317f2a-c574-4966-9946-55e55d521176"/>                            <textFieldExpression><![CDATA[$F{firstName}]]></textFieldExpression>                        </textField>                        <textField isBlankWhenNull="true">                            <reportElement x="100" y="0" width="200" height="13" uuid="36317f2a-c574-4966-9946-55e55d521176"/>                            <textFieldExpression><![CDATA[$F{lastName}]]></textFieldExpression>                        </textField>                        <textField isBlankWhenNull="true">                            <reportElement x="200" y="0" width="300" height="13" uuid="36317f2a-c574-4966-9946-55e55d521176"/>                            <textFieldExpression><![CDATA[$F{postcode}]]></textFieldExpression>                        </textField>                        <textField isBlankWhenNull="true">                            <reportElement x="300" y="0" width="400" height="13" uuid="36317f2a-c574-4966-9946-55e55d521176"/>                            <textFieldExpression><![CDATA[$F{count}]]></textFieldExpression>                        </textField>                    </c:listContents>                </c:list>            </componentElement>        </band>    </detail></jasperReport>

 

 

 

 

[peter.w]

Can we assume that the "user" running the jasper server has permission to read and write to the /target folder?

Regards

Peter

 

[hozawa]

The main problem is that you're using POJO. Output your POJO to a csv file or to a database table and use csv file or database table as an input. POJOs wastes memory and decreases performance.

Also, if there's no "total" in your report, it may be better to break up the input data and multi-thread execution.

[peter.morgan.homeiname.co]

Hello, firstly thank you for your helpful suggestions. I believe I've now solved the issue though, I was setting isIgnorePagination=true in the JRXML file. This prevents the Virtualiser from working. Many thanks again, Pete.