To filter a reference field to sys_user to show members of groups the user is in, you have to use an Advanced reference qualifier rather than a dynamic one. Below shows the dictionary entry for the field and the script include.
Dictionary Entry for Field:

javascript:'sys_idIN' + global.getMembersOfMyGroups(gs.getUserID())
Script Include:

function getMembersOfMyGroups(userSysId) {
var arr_util = new global.ArrayUtil();
var myGroups = arr_util.convertArray(gs.getUser().getMyGroups());
var users = [];
var usrgrpGR = new GlideRecord('sys_user_grmember');
usrgrpGR.addQuery('group', 'IN', myGroups.toString());
usrgrpGR.query();
while (usrgrpGR.next()) {
users.push(usrgrpGR.user.sys_id);
}
return users;
}
