There are quite a number of issues with your code.

• Structure packing (in general, you should not specify it, defaults are consistent between .NET and Win32)
• Ansi vs Unicode
• Error handling (GetLastError is not used by these APIs)

Here's a version that seems to work:

var identity = WindowsIdentity.GetCurrent(); // get some identity

var status = GetNamedSecurityInfo(@"c:\temp\file.txt",
    SE_OBJECT_TYPE.SE_FILE_OBJECT,
    SECURITY_INFORMATION.DACL_SECURITY_INFORMATION | SECURITY_INFORMATION.UNPROTECTED_DACL_SECURITY_INFORMATION,
    IntPtr.Zero,
    IntPtr.Zero,
    out var fileACLHandle,
    IntPtr.Zero,
    IntPtr.Zero);
if (status != 0)
    throw new Win32Exception(status);

var nameTrustee = new TRUSTEE_WITH_NAME
{
    TrusteeForm = TRUSTEE_FORM.TRUSTEE_IS_NAME,
    TrusteeType = TRUSTEE_TYPE.TRUSTEE_IS_USER,
    ptstrName = identity.Name
};

status = GetEffectiveRightsFromAcl(fileACLHandle, ref nameTrustee, out var accessMask);
if (status != 0)
    throw new Win32Exception(status);

Console.WriteLine($"Rights: {accessMask}");

var sid = new byte[identity.User.BinaryLength];
identity.User.GetBinaryForm(sid, 0);

var sidTrustee = new TRUSTEE_WITH_SID
{
    TrusteeForm = TRUSTEE_FORM.TRUSTEE_IS_SID,
    TrusteeType = TRUSTEE_TYPE.TRUSTEE_IS_USER,
    pSid = Marshal.UnsafeAddrOfPinnedArrayElement(sid, 0)
};

status = GetEffectiveRightsFromAcl(fileACLHandle, ref sidTrustee, out accessMask);
if (status != 0)
    throw new Win32Exception(status);

Console.WriteLine($"Rights: {accessMask}");

...

public enum SE_OBJECT_TYPE
{
    SE_UNKNOWN_OBJECT_TYPE,
    SE_FILE_OBJECT,
    SE_SERVICE,
    SE_PRINTER,
    SE_REGISTRY_KEY,
    SE_LMSHARE,
    SE_KERNEL_OBJECT,
    SE_WINDOW_OBJECT,
    SE_DS_OBJECT,
    SE_DS_OBJECT_ALL,
    SE_PROVIDER_DEFINED_OBJECT,
    SE_WMIGUID_OBJECT,
    SE_REGISTRY_WOW64_32KEY
}

[Flags]
public enum SECURITY_INFORMATION
{
    OWNER_SECURITY_INFORMATION = 0x00000001,
    GROUP_SECURITY_INFORMATION = 0x00000002,
    DACL_SECURITY_INFORMATION = 0x00000004,
    SACL_SECURITY_INFORMATION = 0x00000008,
    UNPROTECTED_SACL_SECURITY_INFORMATION = 0x10000000,
    UNPROTECTED_DACL_SECURITY_INFORMATION = 0x20000000,
    PROTECTED_SACL_SECURITY_INFORMATION = 0x40000000,
    PROTECTED_DACL_SECURITY_INFORMATION = unchecked((int)0x80000000)
}

public enum MULTIPLE_TRUSTEE_OPERATION
{
    NO_MULTIPLE_TRUSTEE,
    TRUSTEE_IS_IMPERSONATE
}

public enum TRUSTEE_FORM
{
    TRUSTEE_IS_SID,
    TRUSTEE_IS_NAME,
    TRUSTEE_BAD_FORM,
    TRUSTEE_IS_OBJECTS_AND_SID,
    TRUSTEE_IS_OBJECTS_AND_NAME
}

public enum TRUSTEE_TYPE
{
    TRUSTEE_IS_UNKNOWN,
    TRUSTEE_IS_USER,
    TRUSTEE_IS_GROUP,
    TRUSTEE_IS_DOMAIN,
    TRUSTEE_IS_ALIAS,
    TRUSTEE_IS_WELL_KNOWN_GROUP,
    TRUSTEE_IS_DELETED,
    TRUSTEE_IS_INVALID,
    TRUSTEE_IS_COMPUTER
}

[StructLayout(LayoutKind.Sequential, CharSet = CharSet.Unicode)]
public struct TRUSTEE_WITH_NAME
{
    public IntPtr pMultipleTrustee;
    public MULTIPLE_TRUSTEE_OPERATION MultipleTrusteeOperation;
    public TRUSTEE_FORM TrusteeForm;
    public TRUSTEE_TYPE TrusteeType;
    public string ptstrName;
}

[StructLayout(LayoutKind.Sequential)]
public struct TRUSTEE_WITH_SID
{
    public IntPtr pMultipleTrustee;
    public MULTIPLE_TRUSTEE_OPERATION MultipleTrusteeOperation;
    public TRUSTEE_FORM TrusteeForm;
    public TRUSTEE_TYPE TrusteeType;
    public IntPtr pSid;
}

[DllImport("advapi32", CharSet = CharSet.Unicode)]
public static extern int GetNamedSecurityInfo(string pObjectName, SE_OBJECT_TYPE ObjectType, SECURITY_INFORMATION SecurityInfo, IntPtr pSidOwner, IntPtr pSidGroup, out IntPtr pDacl, IntPtr pSacl, IntPtr pSecurityDescriptor);

[DllImport("advapi32", CharSet = CharSet.Unicode)]
public static extern int GetEffectiveRightsFromAcl(IntPtr pacl, ref TRUSTEE_WITH_NAME pTrustee, out int pAccessRights);

[DllImport("advapi32", CharSet = CharSet.Unicode)]
public static extern int GetEffectiveRightsFromAcl(IntPtr pacl, ref TRUSTEE_WITH_SID pTrustee, out int pAccessRights);