Sitecore Identity was introduced in Sitecore 9.1 and uses the new Federated Authentication functionality. By using the same techniques as Sitecore Identity it's possible to implement a custom identity provider with OpenID Connect.

We start by creating a pipeline. This is an example which is also based on Sitecore Identity:

public class ProjectIdentityProvider : IdentityProvidersProcessor
{
    private readonly IConfigurationRepository configurationRepository;

    private readonly IUrlUtils urlUtils;

    private readonly ICookieManager cookieManager;

    public ProjectIdentityProvider(
        IConfigurationRepository configurationRepository,
        IUrlUtils urlUtils,
        FederatedAuthenticationConfiguration federatedAuthenticationConfiguration,
        ICookieManager cookieManager,
        BaseSettings settings) : base(federatedAuthenticationConfiguration, cookieManager, settings)
    {
        this.configurationRepository = configurationRepository ?? throw new ArgumentNullException(nameof(configurationRepository));
        this.urlUtils = urlUtils ?? throw new ArgumentNullException(nameof(urlUtils));
        this.cookieManager = cookieManager ?? throw new ArgumentNullException(nameof(cookieManager));
    }

    protected override void ProcessCore(IdentityProvidersArgs args)
    {
        var authenticationType = this.GetAuthenticationType();
        var identityProvider = this.GetIdentityProvider();
        var saveSigninToken = identityProvider.TriggerExternalSignOut;

        var oidcOptions = this.SetupOidcOptions(authenticationType, saveSigninToken);

        args.App.UseOpenIdConnectAuthentication(oidcOptions);
    }

    public OpenIdConnectAuthenticationOptions SetupOidcOptions(
        string authenticationType,
        bool saveSigninToken)
    {
        var oidcOptions = new OpenIdConnectAuthenticationOptions
        {
            AuthenticationType = authenticationType,
            AuthenticationMode = AuthenticationMode.Passive,
            MetadataAddress = this.configurationRepository.GetSetting(Constants.Settings.IdentityAccessManagementMetadataAddress),
            ClientId = this.configurationRepository.GetSetting(Constants.Settings.IdentityAccessManagementClientId),
            ClientSecret = this.configurationRepository.GetSetting(Constants.Settings.IdentityAccessManagementClientSecret),
            ResponseMode = OpenIdConnectResponseMode.Query,
            ResponseType = OpenIdConnectResponseType.Code,
            RedeemCode = true,
            Scope = OpenIdConnect.ProjectIdentityScope,
            RequireHttpsMetadata = true,
            Notifications = new OpenIdConnectAuthenticationNotifications
            {
                RedirectToIdentityProvider = this.RedirectToIdentityProviderAsync,
                SecurityTokenValidated = this.SecurityTokenValidatedAsync
            },
            TokenValidationParameters =
            {
                SaveSigninToken = saveSigninToken
            },
            CookieManager = cookieManager
        };

        return oidcOptions;
    }

    protected override string IdentityProviderName => OpenIdConnect.ProjectIdentityProvider;

    protected BaseLog Log { get; }

    public Collection<string> Scopes { get; } = new Collection<string>();

    private Task RedirectToIdentityProviderAsync(
        RedirectToIdentityProviderNotification<OpenIdConnectMessage, OpenIdConnectAuthenticationOptions>
            notification)
    {
        var domain = urlUtils.GetDomain();
        var owinContext = notification.OwinContext;
        var protocolMessage = notification.ProtocolMessage;

        if (protocolMessage.RequestType == OpenIdConnectRequestType.Authentication)
        {
            var redirectUri = this.configurationRepository.GetSetting(Constants.OpenIdConnectOptions.RedirectUri);
            // Make sure the redirectUri goes to the current domain. 
            redirectUri = WebUtil.GetUri(redirectUri, new Uri(domain)).ToString();
            protocolMessage.RedirectUri = redirectUri;
        }

        if (protocolMessage.RequestType == OpenIdConnectRequestType.Logout)
        {
            var postLogoutRedirectUri = this.configurationRepository.GetSetting(Constants.OpenIdConnectOptions.PostLogoutRedirectUri);
            // Make sure the postLogoutRedirectUri goes to the current domain.
            postLogoutRedirectUri = WebUtil.GetUri(postLogoutRedirectUri, new Uri(domain)).ToString();
            protocolMessage.PostLogoutRedirectUri = postLogoutRedirectUri;
            protocolMessage.IdTokenHint = this.GetIdTokenHint(owinContext);
        }
        return Task.CompletedTask;
    }

    private Task SecurityTokenValidatedAsync(SecurityTokenValidatedNotification<OpenIdConnectMessage, OpenIdConnectAuthenticationOptions> notification)
    {
        var identityProvider = this.GetIdentityProvider();
        var identity = notification.AuthenticationTicket.Identity;

        foreach (var current in identityProvider.Transformations)
        {
            current.Transform(identity, new TransformationContext(this.FederatedAuthenticationConfiguration, identityProvider));
        }
        return Task.CompletedTask;
    }
}

The pipeline can be registered the same way as Sitecore Identity:

<configuration xmlns:patch="http://www.sitecore.net/xmlconfig/">
    <sitecore>
        <federatedAuthentication type="Sitecore.Owin.Authentication.Configuration.FederatedAuthenticationConfiguration, Sitecore.Owin.Authentication">
            <identityProvidersPerSites hint="list:AddIdentityProvidersPerSites">
                <mapEntry name="sites with extranet domain" type="Sitecore.Owin.Authentication.Collections.IdentityProvidersPerSitesMapEntry, Sitecore.Owin.Authentication" resolve="true" patch:instead="*[@name='sites with extranet domain']">
                    <sites hint="list">
                        <site>project</site>
                    </sites>
                    <identityProviders hint="list:AddIdentityProvider">
                        <identityProvider ref="federatedAuthentication/identityProviders/identityProvider[@id='ProjectIdentityProvider']" />
                    </identityProviders>
                    <externalUserBuilder type="Sitecore.Owin.Authentication.Services.DefaultExternalUserBuilder, Sitecore.Owin.Authentication" resolve="true">
                        <IsPersistentUser>false</IsPersistentUser>
                    </externalUserBuilder>
                </mapEntry>
            </identityProvidersPerSites>
            <identityProviders>
                <identityProvider id="ProjectIdentityProvider" type="Sitecore.Owin.Authentication.Configuration.DefaultIdentityProvider, Sitecore.Owin.Authentication">
                    <param desc="name">$(id)</param>
                    <param desc="domainManager" type="Sitecore.Abstractions.BaseDomainManager" resolve="true" />
                    <caption>Go to login</caption>
                    <domain>extranet</domain>
                    <triggerExternalSignOut>true</triggerExternalSignOut>
                    <!--list of identity transfromations which are applied to the provider when a user signin-->
                    <transformations hint="list:AddTransformation">
                        <!--SetIdpClaim transformation-->
                        <transformation name="Idp Claim" type="Sitecore.Owin.Authentication.Services.SetIdpClaimTransform, Sitecore.Owin.Authentication" />
                        <!-- If external authentication is configured with "TokenValidationParameters = {SaveSigninToken = true}", this saves the value from "claimsIdentity.BootstrapContext" to the "id_token" claim. -->
                        <transformation name="set id_token claim" type="Sitecore.Owin.Authentication.Services.SaveIdTokenInClaim, Sitecore.Owin.Authentication" />
                    </transformations>
                </identityProvider>
            </identityProviders>
        </federatedAuthentication>
        <pipelines>
            <owin.identityProviders>
                <processor type="Project.Foundation.Identity.IdentityProviders.ProjectIdentityProvider, Project.Foundation.Identity" resolve="true" id="ProjectIdentityProvider">
                    <scopes hint="list">
                        <scope name="openid">openid</scope>
                        <scope name="sitecore.profile">sitecore.profile</scope>
                    </scopes>
                </processor>
            </owin.identityProviders>
        </pipelines>
    </sitecore>
</configuration>

After registering the pipeline you need to set the login page on your site.

<site patch:before="site[@name='website']"
      inherits="website"
      name="project"
      language="nl-NL"
      contentLanguage="nl-NL"
      scheme="https"
      rootPath="/sitecore/content/Project"
      startItem="/Home"
      loginPage="$(loginPath)project/ProjectIdentityProvider" />

If you make a page protected in your project site it will now be redirected to your custom identity provider.

You can find the code on GitHub here: https://gist.github.com/jbreuer/ed69146f63aa1b20058092ab382ee0a7