The |datadirectory| algorithm is located in the System.Data.dll assembly, in the internal System.Data.Common.DbConnectionOptions class. Here it as displayed by ILSpy (note the source it's now available in the reference source repository: https://github.com/Microsoft/referencesource/blob/e458f8df6ded689323d4bd1a2a725ad32668aaec/System.Data.Entity/System/Data/EntityClient/DbConnectionOptions.cs):

internal static string ExpandDataDirectory(string keyword,
                                           string value,
                                           ref string datadir)
{
    string text = null;
    if (value != null && 
        value.StartsWith("|datadirectory|", StringComparison.OrdinalIgnoreCase))
    {
        string text2 = datadir;
        if (text2 == null)
        {
            // 1st step!
            object data = AppDomain.CurrentDomain.GetData("DataDirectory");
            text2 = (data as string);
            if (data != null && text2 == null)
                throw ADP.InvalidDataDirectory();

            if (ADP.IsEmpty(text2))
            {
                // 2nd step!
                text2 = AppDomain.CurrentDomain.BaseDirectory;
            }
            if (text2 == null)
            {
                text2 = "";
            }
            datadir = text2;
        }

        // 3rd step, checks and normalize
        int length = "|datadirectory|".Length;
        bool flag = 0 < text2.Length && text2[text2.Length - 1] == '\\';
        bool flag2 = length < value.Length && value[length] == '\\';
        if (!flag && !flag2)
        {
            text = text2 + '\\' + value.Substring(length);
        }
        else
        {
            if (flag && flag2)
            {
                text = text2 + value.Substring(length + 1);
            }
            else
            {
                text = text2 + value.Substring(length);
            }
        }
        if (!ADP.GetFullPath(text).StartsWith(text2, StringComparison.Ordinal))
            throw ADP.InvalidConnectionOptionValue(keyword);
    }
    return text;
}

So it looks in the current AppDomain data first (by default, there is no "DataDirectory" data defined I believe) and then gets to the current AppDomain base directory. The rest is mostly checks for path roots and paths normalization.