logo

NJP

← Back to News

Removing Duplicates From an Object Array

Import · Aug 18, 2016 · article

NOTE: MY POSTINGS REFLECT MY OWN VIEWS AND DO NOT NECESSARILY REPRESENT THE VIEWS OF MY EMPLOYER, ACCENTURE.

DIFFICULTY LEVEL: INTERMEDIATE
Assumes having taken the class SSNF and has good intermediate level of knowledge and/or familiarity with Scripting in ServiceNow.

Something, as a developer, that I run into quite a bit is the need to remove duplicates from an Object List by a key or a value such as a sys_id. In this article I will describe how to do this with ServiceNow Scripting and JavaScript.

This a very useful type of functionality when dealing with any size object array. In the following example I will build a list (array) of cmdb_ci objects, build a key for each object in the list, and then use a function to remove all of the duplicates. The key is simply an attempt to build a unique identifier. This identifier can be a sys_id, or some combination of values that will allow the code to identify a unique record.

var incidentRecords = new GlideRecord('incident');
incidentRecords.addNotNullQuery('cmdb_ci');
incidentRecords.addActiveQuery();
incidentRecords.query();

var cmdbList = [];

// The following looping code is a GlideRecord inside of a GlideRecord.
// This is normally a BAD way of doing this, but I want
// to make SURE I have duplicates in the Object List!

while (incidentRecords.next()) {
    var ciRecords = new GlideRecord('cmdb_ci');
    if (ciRecords.get('sys_id', incidentRecords.getValue('cmdb_ci'))) {
        cmdb_ci = {};
        cmdb_ci.serial_number = ciRecords.getValue('serial_number');
        cmdb_ci.name = ciRecords.getValue('name');
        cmdb_ci.className = ciRecords.getValue('sys_class_name');
        cmdb_ci.install_status = ciRecords.getValue('install_status');
        cmdb_ci.key = cmdb_ci.serial_number + ':' + cmdb_ci.name + ':' + cmdb_ci.className;
        cmdbList.push(cmdb_ci);
    }
}

gs.info('BEFORE: {0}\n{1}', [cmdbList.length, JSON.stringify(cmdbList)]);

cmdbList = uniqueObjectList(cmdbList);

gs.info('AFTER: {0}\n{1}', [cmdbList.length, JSON.stringify(cmdbList)]);

function uniqueObjectList(dedupList) {
    for (var i = 0; i < dedupList.length; i++) {
        for (var j = i + 1; j < dedupList.length; j++) {
            if (dedupList[j].key == dedupList[i].key) {
                dedupList.splice(j, 1);
                --j;
            }
        }
    }
    return dedupList;
}

Results:

*** Script: BEFORE: 18.0[{"serial_number":null,"name":"Saints and Sinners Bingo","className":"cmdb_ci_spkg","install_status":"1","key":"null:Saints and Sinners Bingo:cmdb_ci_spkg"},{"serial_number":"L3CD257","name":"IBM-T42-DLG","className":"cmdb_ci_computer","install_status":null,"key":"L3CD257:IBM-T42-DLG:cmdb_ci_computer"},{"serial_number":null,"name":"Windows XP Hotfix (SP2) Q817606","className":"cmdb_ci_spkg","install_status":"1","key":"null:Windows XP Hotfix (SP2) Q817606:cmdb_ci_spkg"},{"serial_number":null,"name":"WeatherBug","className":"cmdb_ci_spkg","install_status":"1","key":"null:WeatherBug:cmdb_ci_spkg"},{"serial_number":null,"name":"EFOWEB","className":"cmdb_ci_computer","install_status":"1","key":"null:EFOWEB:cmdb_ci_computer"},{"serial_number":null,"name":"MailServerUS","className":"cmdb_ci_server","install_status":"1","key":"null:MailServerUS:cmdb_ci_server"},{"serial_number":null,"name":"WEBSERVER","className":"cmdb_ci_computer","install_status":"1","key":"null:WEBSERVER:cmdb_ci_computer"},{"serial_number":null,"name":"FileServerFloor2","className":"cmdb_ci_server","install_status":"1","key":"null:FileServerFloor2:cmdb_ci_server"},{"serial_number":"L3BB911","name":"JEMPLOYEE-IBM","className":"cmdb_ci_computer","install_status":"1","key":"L3BB911:*JEMPLOYEE-IBM:cmdb_ci_computer"},{"serial_number":null,"name":"Sales Force Automation","className":"cmdb_ci_service","install_status":"1","key":"null:Sales Force Automation:cmdb_ci_service"},{"serial_number":null,"name":"Sales Force Automation","className":"cmdb_ci_service","install_status":"1","key":"null:Sales Force Automation:cmdb_ci_service"},{"serial_number":null,"name":"SAP Controlling","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Controlling:cmdb_ci_service"},{"serial_number":null,"name":"SAP Financial Accounting","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Financial Accounting:cmdb_ci_service"},{"serial_number":null,"name":"SAP Human Resources","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Human Resources:cmdb_ci_service"},{"serial_number":null,"name":"SAP Materials Management","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Materials Management:cmdb_ci_service"},{"serial_number":null,"name":"SAP Sales and Distribution","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Sales and Distribution:cmdb_ci_service"},{"serial_number":"cx320-4001-21a","name":"nyc rac nas200","className":"cmdb_ci_msd","install_status":"1","key":"cx320-4001-21a:nyc rac nas200:cmdb_ci_msd"},{"serial_number":null,"name":"EXCH-SD-05","className":"cmdb_ci_email_server","install_status":"1","key":"null:EXCH-SD-05:cmdb_ci_email_server"}]** Script: AFTER: 17.0[{"serial_number":null,"name":"Saints and Sinners Bingo","className":"cmdb_ci_spkg","install_status":"1","key":"null:Saints and Sinners Bingo:cmdb_ci_spkg"},{"serial_number":"L3CD257","name":"IBM-T42-DLG","className":"cmdb_ci_computer","install_status":null,"key":"L3CD257:IBM-T42-DLG:cmdb_ci_computer"},{"serial_number":null,"name":"Windows XP Hotfix (SP2) Q817606","className":"cmdb_ci_spkg","install_status":"1","key":"null:Windows XP Hotfix (SP2) Q817606:cmdb_ci_spkg"},{"serial_number":null,"name":"WeatherBug","className":"cmdb_ci_spkg","install_status":"1","key":"null:WeatherBug:cmdb_ci_spkg"},{"serial_number":null,"name":"EFOWEB","className":"cmdb_ci_computer","install_status":"1","key":"null:EFOWEB:cmdb_ci_computer"},{"serial_number":null,"name":"MailServerUS","className":"cmdb_ci_server","install_status":"1","key":"null:MailServerUS:cmdb_ci_server"},{"serial_number":null,"name":"WEBSERVER","className":"cmdb_ci_computer","install_status":"1","key":"null:WEBSERVER:cmdb_ci_computer"},{"serial_number":null,"name":"FileServerFloor2","className":"cmdb_ci_server","install_status":"1","key":"null:FileServerFloor2:cmdb_ci_server"},{"serial_number":"L3BB911","name":"*JEMPLOYEE-IBM","className":"cmdb_ci_computer","install_status":"1","key":"L3BB911:*JEMPLOYEE-IBM:cmdb_ci_computer"},{"serial_number":null,"name":"Sales Force Automation","className":"cmdb_ci_service","install_status":"1","key":"null:Sales Force Automation:cmdb_ci_service"},{"serial_number":null,"name":"SAP Controlling","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Controlling:cmdb_ci_service"},{"serial_number":null,"name":"SAP Financial Accounting","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Financial Accounting:cmdb_ci_service"},{"serial_number":null,"name":"SAP Human Resources","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Human Resources:cmdb_ci_service"},{"serial_number":null,"name":"SAP Materials Management","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Materials Management:cmdb_ci_service"},{"serial_number":null,"name":"SAP Sales and Distribution","className":"cmdb_ci_service","install_status":"1","key":"null:SAP Sales and Distribution:cmdb_ci_service"},{"serial_number":"cx320-4001-21a","name":"nyc rac nas200","className":"cmdb_ci_msd","install_status":"1","key":"cx320-4001-21a:nyc rac nas200:cmdb_ci_msd"},{"serial_number":null,"name":"EXCH-SD-05","className":"cmdb_ci_email_server","install_status":"1","key":"null:EXCH-SD-05:cmdb_ci_email_server"}

]

If you look careful you will see that there is a duplicate of the following entry:

{"serial_number":null,"name":"Sales Force Automation","className":"cmdb_ci_service","install_status":"1","key":"null:Sales Force Automation:cmdb_ci_service"},

{"serial_number":null,"name":"Sales Force Automation","className":"cmdb_ci_service","install_status":"1","key":"null:Sales Force Automation:cmdb_ci_service"},

and it has been removed.

Okay, so let's refactor our new uniqueObjectList function a bit.

var incidentRecords = new GlideRecord('incident');
incidentRecords.addNotNullQuery('cmdb_ci');
incidentRecords.addActiveQuery();
incidentRecords.query();

var cmdbList = [];

// The following looping code is a GlideRecord inside of a GlideRecord.
// This is normally a BAD way of doing this, but I want
// to make SURE I have duplicates in the Object List!

while (incidentRecords.next()) {
    var ciRecords = new GlideRecord('cmdb_ci');
    if (ciRecords.get('sys_id', incidentRecords.getValue('cmdb_ci'))) {
        cmdb_ci = {};
        cmdb_ci.serial_number = ciRecords.getValue('serial_number');
        cmdb_ci.name = ciRecords.getValue('name');
        cmdb_ci.className = ciRecords.getValue('sys_class_name');
        cmdb_ci.install_status = ciRecords.getValue('install_status');
        cmdb_ci.key = cmdb_ci.serial_number + ':' + cmdb_ci.name + ':' + cmdb_ci.className;
        cmdbList.push(cmdb_ci);
    }
}

gs.info('BEFORE: {0}\n{1}', [cmdbList.length, JSON.stringify(cmdbList)]);

cmdbList = uniqueObjectList(cmdbList, 'key');

gs.info('AFTER: {0}\n{1}', [cmdbList.length, JSON.stringify(cmdbList)]);

function uniqueObjectList (dedupList, field) {
    for (var i = 0; i < dedupList.length; i++) {
        for (var j = i + 1; j < dedupList.length; j++) {
            if (dedupList[j][field] == dedupList[i][field]) {
                dedupList.splice(j, 1);
                --j;
            }
        }
    }
    return dedupList;
}

The results should be exactly the same.

You might want to put this into a Script Include function library (utils) of some sort for ease of use, and to minimize any maintenance. Just a thought.

Despite the disdain I run into from other programmers working with "real" languages (i.e. C#, Java), I feel JavaScript can be just as powerful in its own way! And lest you sniff at that comment, let me point out that I have done C#/.Net development since .Net 0.9 Beta, and I took a class on OOP from Bjarne Stroustrup himself when he had just started touring to talk about C++! Go Ada! Alas, I date myself. Each language has it's uses!

Enjoy!

Steven Bell.

If you find this article helps you, don't forget to log in and mark it as "Helpful"!

image

Originally published on: 08-18-2016 09:51 PM

I updated the code and brought the article into alignment with my new formatting standard.

View original source

https://www.servicenow.com/community/developer-blog/community-code-snippets-removing-duplicates-from-an-object-array/ba-p/2267181