This is a handy little script that allows you to prevent a parent record (such as an incident or a problem) from being closed if there are still open child tasks.

First, add a client callable script includes with the name childTasks

var childTasks = Class.create();
childTasks.prototype = Object.extendsObject(AbstractAjaxProcessor, {

    activeTasks:function() { 

        var result = [];        
        var whichTable = this.getParameter('sysparm_table');
        var whichField = this.getParameter('sysparm_field');
        var whichValue = this.getParameter('sysparm_value');

        var gr = new GlideRecord(whichTable);
        gr.addEncodedQuery(whichField+'.sys_id='+whichValue);
        gr.query();

        while(gr.next()){
            if(gr.active){result.push(gr.number.toString());}
        }
        return result.toString(); 
    },

    type: 'childTasks'
});

Then update the Resolve button's UI Action script to call this server side code via GlideAjax

For Incident, this is...

function resolveIncident(){

    var ga = new GlideAjax('childTasks');
    ga.addParam('sysparm_name', 'activeTasks');
    ga.addParam('sysparm_table', 'incident_task');
    ga.addParam('sysparm_field', 'incident');
    ga.addParam('sysparm_value', g_form.getUniqueValue());
    ga.getXMLAnswer(activeTasks);

    function activeTasks(response) {

        var outstandingTasks = response;

        if(outstandingTasks!=''){
            g_form.addErrorMessage('You must close the following related tasks before you can resolve this record: ' + outstandingTasks.split(',').join(', '));
        }else{
            //There are no active child tasks so proceed
            //Set the 'Incident state' and 'State' values to 'Resolved', and display mandatory fields
            g_form.setValue('incident_state', 6);
            g_form.setValue('state', 6);
            g_form.setValue('resolved_by', g_user.userID);

            gsftSubmit(null, g_form.getFormElement(), 'resolve_incident'); //MUST call the 'Action name' set in this UI Action
        }
    }
}

//Code that runs without 'onclick'
//Ensure call to server-side function with no browser errors
if (typeof window == 'undefined')
    serverResolve();

function serverResolve(){
    current.incident_state = IncidentState.RESOLVED;
    current.state = IncidentState.RESOLVED;
    current.resolved_by = gs.getUserID();
    current.update();
}

Note, only the first 15 lines (up until the }else{ are actually new. Everything else is the OOB code from ServiceNow.

This code evaluates whether there are any open tasks before proceeding with the resolved actions.

For problem this is...

function onResolve() {

    var ga = new GlideAjax('childTasks');
    ga.addParam('sysparm_name', 'activeTasks');
    ga.addParam('sysparm_table', 'problem_task');
    ga.addParam('sysparm_field', 'problem');
    ga.addParam('sysparm_value', g_form.getUniqueValue());
    ga.getXMLAnswer(activeTasks);

    function activeTasks(response) {

        var outstandingTasks = response;

        if(outstandingTasks!=''){
            g_form.addErrorMessage('You must close the following related tasks before you can resolve this record: ' + outstandingTasks.split(',').join(', '));
        }else{
            if (!g_form.hasField("state") || !g_form.hasField("resolution_code")) {
                getMessage('Cannot resolve the Problem as atleast one of the following fields are not visible: \'State\' \'Resolution code\'', function(msg) {
                    g_form.addErrorMessage(msg);
                });
                return false;
            }
            g_form.setValue("state", g_scratchpad.STATE.RESOLVED);
            g_form.setValue("resolution_code", g_scratchpad.RESOLUTION_CODES.FIX_APPLIED);
            g_form.save();
        }
    }
}

Have fun