Use the following sample in domain.xml (under config folder) in GlassFish. Change the digest-algorithm and encoding accordingly.
<security-service>
<auth-realm classname="com.sun.enterprise.security.auth.realm.jdbc.JDBCRealm" name="jdbcRealm">
<property name="jaas-context" value="jdbcRealm"></property>
<property name="datasource-jndi" value="jdbc/srack"></property>
<property name="user-table" value="users"></property>
<property name="user-name-column" value="username"></property>
<property name="password-column" value="password"></property>
<property name="group-table" value="groups"></property>
<property name="group-name-column" value="groupname"></property>
<property name="digest-algorithm" value="MD5"></property>
<property name="encoding" value="Base64"></property>
<property name="assign-groups" value=""></property>
</auth-realm>
............
More config here
.............
