BatchUploader and ResponseTransformer

[Developer12145]

Hi Isomorphic,

please see the following testcase:

Setup:
SmartClient Version: v12.0p_2020-01-09/PowerEdition Deployment (built 2020-01-09)

BuiltInDS.java

package com.smartgwt.sample.client;

import java.util.List;
import java.util.Map;

import com.google.gwt.core.client.EntryPoint;
import com.google.gwt.core.client.JavaScriptObject;
import com.smartgwt.client.core.KeyIdentifier;
import com.smartgwt.client.data.DSRequest;
import com.smartgwt.client.data.DSResponse;
import com.smartgwt.client.data.DataSource;
import com.smartgwt.client.data.Record;
import com.smartgwt.client.data.RecordList;
import com.smartgwt.client.data.ResponseTransformer;
import com.smartgwt.client.types.PartialCommitOption;
import com.smartgwt.client.util.JSOHelper;
import com.smartgwt.client.util.Page;
import com.smartgwt.client.util.PageKeyHandler;
import com.smartgwt.client.util.SC;
import com.smartgwt.client.widgets.BatchUploader;
import com.smartgwt.client.widgets.grid.ListGridField;
import com.smartgwt.client.widgets.grid.ListGridRecord;
import com.smartgwt.client.widgets.layout.VStack;

/**
 * Entry point classes define <code>onModuleLoad()</code>.
 */
public class BuiltInDS implements EntryPoint {

    /**
     * This is the entry point method.
     */
    public void onModuleLoad() {
        KeyIdentifier debugKey = new KeyIdentifier();
        debugKey.setCtrlKey(true);
        debugKey.setKeyName("D");

        Page.registerKey(debugKey, new PageKeyHandler() {
            public void execute(String keyName) {
                SC.showConsole();
            }
        });

        VStack vStack = new VStack();
        vStack.setLeft(175);
        vStack.setTop(75);
        vStack.setWidth("70%");
        vStack.setMembersMargin(20);

        BatchUploader batchUploader = new BatchUploader();
        batchUploader.setUploadDataSource(DataSource.get("animals"));
        batchUploader.setPartialCommit(PartialCommitOption.RETAIN);
        batchUploader.setUploadDelimiter(";");
        ListGridField commonName = new ListGridField("commonName");
        ListGridField scientificName = new ListGridField("scientificName");
        ListGridField moreInformation = new ListGridField("moreInformation");
        batchUploader.setGridFields(commonName, scientificName, moreInformation);
        batchUploader.draw();
        DataSource.get("batchUpload", null, new ResponseTransformer() {

            @SuppressWarnings("unchecked")
            @Override
            protected void transformResponse(DSResponse response, DSRequest request, Object data) {
                // Problem 1:
                // TRY_NO.1
                // Log "moreInformation" BEFORE
                SC.logWarn(((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0))
                        .get("moreInformation") != null
                                ? ((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0))
                                        .get("moreInformation").toString()
                                : "null");

                // Add value fo "moreInformation"
                ((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0)).put("moreInformation", "Y");

                // Log "moreInformation" AFTER
                SC.logWarn(((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0))
                        .get("moreInformation") != null
                                ? ((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0))
                                        .get("moreInformation").toString()
                                : "null");

                // TRY_NO.2
                List<Map<String, Object>> gridRows = (List<Map<String, Object>>) response.getDataAsMap().get("gridRows");
                // Log "moreInformation" BEFORE
                SC.logWarn(gridRows.get(0).get("moreInformation") != null ? gridRows.get(0).get("moreInformation").toString() : "null");

                // Add value fo "moreInformation"
                gridRows.get(0).put("moreInformation", "Y");

                // Log "moreInformation" AFTER
                SC.logWarn(gridRows.get(0).get("moreInformation") != null ? gridRows.get(0).get("moreInformation").toString() : "null");

                // Doesn't work
                response.setAttribute("gridRows", gridRows);
                // Doesn't work either
                JavaScriptObject jsoArray = JSOHelper.convertMapToJavascriptObject(gridRows.get(0));
                Record[] recordsA = Record.convertToRecordArray(jsoArray);
                ListGridRecord[] lGrecords = new ListGridRecord[1];
                lGrecords[0] = new ListGridRecord(recordsA[0].getJsObj());
                response.setAttribute("operationType", "fetch");
                response.setData(lGrecords);

                // Problem 2:
                // works
                RecordList records = response.getDataAsRecordList();
                // EXCEPTION: com.google.gwt.core.client.JavaScriptException
                int length = response.getDataAsRecordList().getLength();
            }
        });

    }
}

BuiltInDS.html

<!DOCTYPE html>

<html>
  <head>
    <meta http-equiv="content-type" content="text/html; charset=UTF-8">
    <!--                                           -->
    <!-- Any title is fine                         -->
    <!--                                           -->
    <title>BuiltInDS</title>

    <!-- IMPORTANT : You must set the variable isomorphicDir to [MODULE_NAME]/sc/ so that the SmartGWT resource are 
      correctly resolved -->    
    <script> var isomorphicDir = "builtinds/sc/"; </script>

    <!--                                           -->
    <!-- This script loads your compiled module.   -->
    <!-- If you add any GWT meta tags, they must   -->
    <!-- be added before this line.                -->
    <!--                                           -->      
    <script type="text/javascript" language="javascript" src="builtinds/builtinds.nocache.js"></script>

    <!-- The following script is required if you're running (Super)DevMode and are using module
         definitions that contain <script> tags.  Normally, this script is loaded automatically
         by builtinds.nocache.js above, but this isn't possible when (Super)DevMode is running.
         Note: it should not create any issue to always load it below (even if already loaded). -->
    <script type="text/javascript" language="javascript" src="builtinds/loadScriptTagFiles.js"></script>

  </head>

  <!--                                           -->
  <!-- The body can have arbitrary html, or      -->
  <!-- you can leave the body empty if you want  -->
  <!-- to create a completely dynamic UI.        -->
  <!--                                           -->
  <body>

    <!--load the datasources-->
    <script src="builtinds/sc/DataSourceLoader?dataSource=supplyItem,animals,employees,batchUpload"></script>

    <!-- OPTIONAL: include this if you want history support -->
    <iframe src="javascript:''" id="__gwt_historyFrame" tabIndex='-1' style="position:absolute;width:0;height:0;border:0"></iframe>

  </body>
</html>

animals.ds.xml

<DataSource
    ID="animals"
    serverType="sql"
    tableName="animals"
    testFileName="animals.data.xml"
>
    <fields>
        <field name="commonName"      title="Animal"             type="text" >
     </field>
        <field name="scientificName"  title="Scientific Name"    type="text"  primaryKey="true"  required="true"/>
        <field name="lifeSpan"        title="Life Span"          type="integer"/>
        <field name="status"          title="Endangered Status"  type="text">
            <valueMap>
                <value>Threatened</value>
                <value>Endangered</value>
                <value>Not Endangered</value>
                <value>Not currently listed</value>
                <value>May become threatened</value>
                <value>Protected</value>
            </valueMap>
        </field>
        <field name="diet"            title="Diet"               type="text"/>
        <field name="information"     title="Interesting Facts"  type="text"  length="1000"/>
        <field name="moreInformation" customSelectExpression="1"/>
        <field name="picture"         title="Picture"            type="image" detail="true"
               imageURLPrefix="/isomorphic/system/reference/inlineExamples/tiles/images/"/>


    </fields>
</DataSource>

batchUpload.ds.xml

<DataSource ID="batchUpload">
    <operationBindings>
        <operationBinding operationType="add" operationId="upload" serverMethod="batchUpload">
            <explanation>First do: boolean conversions, date conversions and list conversions. Then validate and upload data.</explanation>
        </operationBinding>
    </operationBindings>
    <serverObject ID="batchUpload" className="com.smartgwt.sample.server.listener.BatchUploadDMI" dropExtraFields="false">
        <visibleMethods>
            <method name="batchUpload" />
            <method name="wipeData" />
        </visibleMethods>
    </serverObject>
</DataSource>

BatchUploadDMI.java

package com.smartgwt.sample.server.listener;

import java.util.Map;

import com.isomorphic.datasource.DSRequest;
import com.isomorphic.datasource.DSResponse;
import com.isomorphic.tools.BatchUpload;

public class BatchUploadDMI {

    public DSResponse batchUpload(DSRequest dsRequest) throws Exception {
        BatchUpload batchUpload = new BatchUpload();

        // parse data and get the result Map
        DSResponse response = batchUpload.parseUploadData(dsRequest);
        Map<?, ?> respData = response.getDataMap();

        // do not proceed to validation if parsing failed
        if (respData.containsKey("errorMessage"))
            return response;

        DSResponse validatedData = batchUpload.validateUploadData(response);
        return validatedData;
    }
}

You can find Problem1 and Problem2 in BuiltInDS.java

Problem 1:
How can I put the fakeField "moreInformation" in the Response, so that it is also delivered to the client?

Please see my above trys.


TRY_NO.1:
I tried to put the "moreInformation","Y" to the dataMap by doing the following:

    ((Map<String, Object>) ((List<Map<String, Object>>) response.getDataAsMap().get("gridRows")).get(0)).put("moreInformation", "Y");

If You run the code, the log says null null. So I guess I always receive a new dataMap, am I right here?
So that doesnt work

TRY_NO.2:

List<Map<String, Object>> gridRows = (List<Map<String, Object>>) response.getDataAsMap().get("gridRows");
                gridRows.get(0).put("moreInformation", "Y");

with this code (obviously) the map holds the value afterwards. (See log)
But how do I get it back in the Response?
I tried:

response.setAttribute("gridRows", gridRows);

and

        JavaScriptObject jsoArray = JSOHelper.convertMapToJavascriptObject(gridRows.get(0));
                Record[] recordsA = Record.convertToRecordArray(jsoArray);
                ListGridRecord[] lGrecords = new ListGridRecord[1];
                lGrecords[0] = new ListGridRecord(recordsA[0].getJsObj());
                response.setAttribute("operationType", "fetch");
                response.setData(lGrecords);

but neither of both worked


TRY_NO.3:

i simply tried

        for (Record r : response.getData()) {
                    r.setAttribute("moreInformation", "Y");
                }

Problem 2:
I would have prefered to work with

RecordList records = response.getDataAsRecordList();

and then calling

setAttribute("moreInformation", "Y")

But here is something wrong now I think. "records" is not null BUT if I call

getLength()

on it, it throws an com.google.gwt.core.client.JavaScriptException.

Can You please give me some advice on how to get that field to the client within the transformResponse method?

Thanks in advance,
Kind Regards

[Isomorphic]

Although you can see the format of the BatchUploader response in the RCP tab, it's actually not a documented format, so it would be invalid to try to modify it. Instead, you can put any additional attributes on the records once they have been placed in the grid.

[Blama]

Hi Isomorphic,

we did like you suggested in #2 and here with with addPreviewShownHandler(), but this has the disadvantage of not supporting taking action on records edited in the Preview ListGrid.
If the handler is supposed to fire here as well, this does not happen.

We would also be fine with modifing a undocumented format and adjusting code from time to time if necessary, if this worked in any of the ways the OP suggested.

Best regards
Blama

[Isomorphic]

Sorry, could you clarify. You wait until the grid is populated with the preview and then you add additional data as needed - what's the issue here? The previewShownHandler doesn't fire per-record, it fires when the preview is shown, so it shouldn't matter which were edited or not.

[Blama]

Hi Isomorphic,

in order to explain what we are doing:
We used Hilite + ValidationContext.setResultingValue() to create "warnings" via Hilite for possible duplicates in the Upload preview window.

The flow with PreviewShownHandler for us is this:

• Upload CSV
• (Expensive) serverside Validator with validationContext.setResultingValue() on columns for hidden ListGridFields in order to mark columns are to be warned
• ListGrid with Hilites
• PreviewShownHandler to modify records for correct warning display

This is working perfect for the click of the Upload button.
But if the user is editing in the ListGrid, validation is called again and validationContext.setResultingValue() might have a different result, but the PreviewShownHandler needed to modify records depending on the value in the hidden ListGridField does not run.

Possibilities to solve this I can think of:

• You: Use ResponseTransformer (nicest and clearest solution IMHO)
• You: Make PreviewShownHandler run every time
• You: Add another handler for validation-runs
• You: Have the possibility of validationContext.setResultingValue(String columnName, Object value) in order to modify other columns from a Validator as well
• We: Run (expensive) serverside Validator for all columns that PreviewShownHandler would modify and do the data preparation serverside

Best regards
Blama

[Isomorphic]

We've made changes so that PreviewShownHandler would be called after partial commit as well. You may download latest nightly build and try it out.

[Blama]

Hi Isomorphic,

thank you, the change is doing what you say in v12.0p_2020-01-27 in the modified sample below.
But this does not solve the issue. You are running previewShown now also after the Commit click, but as written in #5 the issue is with edited records and therefore re-run serverside validation. We need to react here on field value changes coming from validationContext.setResultingValue() that just ran for the edited row in question.

isc.BatchUploader.create({
    ID: "uploader",
    height: 400,
    width: "100%",
    uploadDataSource: supplyItemCustom,
    gridProperties: {
        height: 200
    },
[B]    previewShown: function(grid) {
        isc.logWarn("Called previewShown")
    }[/B]
});

isc.DynamicForm.create({
    ID: "partialCommitsForm",
    items: [{
        name: "partialCommits",
        wrapTitle: false,
        title: "Partial Commit Mode",
        type: "select",
        defaultValue: "prompt",
        valueMap: {
            allow: "Allow",
            prevent: "Prevent",
            prompt: "Prompt",
            retain: "Retain"
        },
        changed: function(form, item, value) {
            uploader.partialCommit = value;
        }
    }]
});

isc.VStack.create({
    width: "100%",
    layoutMargin: 10,
    membersMargin: 10,
    members: [
        partialCommitsForm,
        uploader
    ]
});

Best regards
Blama

[Blama]

Hi Isomorphic,

any update on this one?

Best regards
Blama

[Isomorphic]

Hi Blama
Sorry for the delay.

We are looking at this and contemplating whether to add a standard ListGrid notification method that fires on validation completion (from user edit) which would allow you to run your custom code on validation completion (success or failure, client or server side).

However, this may not be necessary.
Can we get a clarification on your use case / scenario?

As we understand it:

Setup:
- You have a BatchUploader with a preview grid showing some set of fields (for argument let's call them "field1", "field2" and "moreInformation")
- The DataSource potentially has other fields that aren't visible in the grid ("field4", say)
- The "moreInformation" field is not part of your DataSource, and will be used to display warnings to the user in some cases
- You have Hilites specified on this field to show a custom appearance when such a warning is present

Usage:
- The user uploads a csv file which is validated on the server, and populates this preview grid as expected
- The user makes edits to some cell ("field1", say), and validation occurs automatically, as soon as they're done editing the cell
- On the server some, condition is met that you want to show to the user as a warning in the "moreInformation" field
- You are currently attempting to achieve this by having the server side ValidationContext.setResultingValue() change the value of some field that isn't visible in the grid ("field4"). [I'm assuming you've got this part successfully working, so the value is being passed to the client?]
- ... and then having the (visible) "moreInformation" field be populated based on that result, and the appropriate hilight applied to it [this is the part that isn't working, as you need an entry point to set the value of this other field once the validation has completed]

Is that correct or are we misunderstanding the situation?

If the above is accurate...
We're assuming there's a good reason not to just have the "moreInformation" field be defined in the DataSource and directly populated by "ValidationContext.setResultingValue()" rather than using this hidden "field5", correct?

So as an alternative approach can the "moreInformation" field value be dynamically generated on the client by a cellFormatter? This formatter could look at other attributes on the record (such as "field5" and return an an appropriate warning message for the user)
Cell Hilite definitions use criteria to determine whether or not they should show -- could you similarly have the hilite applied to the "moreInformation" have criteria that consult this hidden field value?

cellFormatters and hilites shoul be reapplied dynamically whenever the cell/row is refreshed, so it seems likely this will give you what you're after without requiring an explicit "validationComplete" entry point to apply the value to the record dynamically.


Regards
Isomorphic Software