Such functionality can be achieved only by code modification, please check
dhtmlxgrid.js, line 2786
if (0!=id)
str.push(self._serializeRow(self.rowsAr[id],i));
// next line need to be added to include only checked branches
if (this.cells(id,INDEX).isChecked())
for(var i=0; i<z.childs.length; i++)
str.push(f(z.childs[i].id,f));