package main import ( "bufio" "fmt" "io" "log" "math/rand" "os" "os/exec" "regexp" "runtime" "strconv" "strings" "time" "github.com/antchfx/xmlquery" "github.com/grmrgecko/go-freeipa" UNIXAccounts "github.com/grmrgecko/go-unixaccounts" "github.com/prometheus/client_golang/prometheus" ) // Creates a metric and appends it to the to the available metrics if enabled. func (e *FreeIPAExporter) NewMetric(metricName string, docString string, t prometheus.ValueType, value func() (float64, error)) { // If metric is disabled, stop here. for _, metric := range e.config.DisabledMetrics { if metric == metricName { return } } // Create info for this metric. info := metricInfo{ Desc: prometheus.NewDesc( prometheus.BuildFQName(namespace, "config", metricName), docString, nil, nil, ), Type: t, Value: value, } // Add metric to list. e.metrics = append(e.metrics, info) } // Sets up the exporter with all needed metrics for FreeIPA. func (e *FreeIPAExporter) setupMetrics() { // Setup basic metrics. e.up = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "up", Help: "Was the last scrape of FreeIPA successful.", }) e.totalScrapes = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Name: "scrapes_total", Help: "Current total HAProxy scrapes.", }) e.totalFailures = prometheus.NewCounter(prometheus.CounterOpts{ Namespace: namespace, Name: "failures_total", Help: "Number of errors while scapping metrics.", }) e.failedTests = prometheus.NewGauge(prometheus.GaugeOpts{ Namespace: namespace, Name: "freeipa_failed_tests", Help: "Number of failed tests in the most recent scrape.", }) // Test: Ensure that kerberos authentication works with kinit and klist. e.NewMetric("krb5_auth", "Kerberos can authenticate.", prometheus.GaugeValue, func() (float64, error) { // Specific cache file for tokens to ensure we're not succeeding due to previous check. krb5CacheFile := fmt.Sprintf("/tmp/krb5_cache_%d", rand.Int()) // Remove file when test is done. defer os.Remove(krb5CacheFile) // Get pricipal with realm for authentication. krb5Principal := e.config.Krb5Principal + "@" + e.config.Krb5Realm // Authenticate with kinit. cmd := exec.Command(app.config.KInitBin, "-kt", app.config.Krb5KeytabPath, "-c", krb5CacheFile, krb5Principal) // Run kinit, which will error if exit code is not 0 as expected. _, err := cmd.Output() if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // Verify the cache file contains the token with klist. cmd = exec.Command(app.config.KListBin, "-c", krb5CacheFile) // Run klist, which will error if exit code is not 0 as expected. _, err = cmd.Output() if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // Both tests suceeded, return 1. return 1, nil }) // Test: The number of workers configured for the directory server should equal number of cores. e.NewMetric("krb5_workers", "Workers match processors.", prometheus.GaugeValue, func() (float64, error) { // Open the kerberos server configuration file. f, err := os.Open(app.config.Krb5SysConfigPath) if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // Close file after test is done. defer f.Close() // Scan the file for each line. scanner := bufio.NewScanner(f) scanner.Split(bufio.ScanLines) // Default to zero workers. workers := 0 // The workers are defined as `-w WORKERS` in the cli arguments. rxWorkers := regexp.MustCompile(`=.*-w\s*([0-9]+)`) // Scan each line for the number of workers. for scanner.Scan() { line := scanner.Text() // If line is the arguments config, parse it. if strings.HasPrefix(line, "KRB5KDC_ARGS") { // Parse line for number of workers. match := rxWorkers.FindStringSubmatch(line) if len(match) == 2 { workers, _ = strconv.Atoi(match[1]) } } } // Check number of workers configured against number of CPU cores. if workers != runtime.NumCPU() { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("number of workers does not match CPU cores") } // If successful, return 1. return 1, nil }) /* Test: The DNA range specify the starting user ID, there needs to be at least one master with a range set. */ e.NewMetric("dna_range", "DNA range is defined.", prometheus.GaugeValue, func() (float64, error) { // Pull from FreeIPA API all DNA ranges found. params := make(map[string]interface{}) req := freeipa.NewRequest( "idrange_find", []interface{}{""}, params, ) res, err := e.conn.Do(req) if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // If no DNA ranges found, fail. if res.Result.Count == 0 { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("no DNA ID range found") } // Success, return 1. return 1, nil }) /* Test: This ensures that groups specified in the config have specific members. By default, we define this check to verify the ipaapi user is a member of the apache group. This is critical for security and access to cache. */ e.NewMetric("group_members", "Group members are as expected.", prometheus.GaugeValue, func() (float64, error) { // Get UNIX account list. accounts, err := UNIXAccounts.NewUNIXAccounts() if err != nil { e.failedTests.Inc() return 0, err } // Check each ground member configuration. for _, check := range e.config.GroupMembers { // Find the group that matches the name configured. group := accounts.GroupWithName(check.Name) // If group not found, fail. if group == nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("unable to find group with name: %s", check.Name) } // Get all users in this group. users := accounts.UsersInGroup(group) // Check each member configured to verify they are a group memebr. for _, member := range check.Members { // Check all users in this group to see if they are this member. userIsMember := false for _, user := range users { if user.Name == member { userIsMember = true } } // If member isn't a user in the group, fail. if !userIsMember { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("user %s should be a member of %s", member, check.Name) } } } // If we reached this point, no test failed. return 1, nil }) // Test: Confirm the shared secret between tomcat and Apache match. e.NewMetric("proxy_secret", "Proxy secret is configured.", prometheus.GaugeValue, func() (float64, error) { // Open the tomcat server configuration file. xmlF, err := os.Open(app.config.PKITomcatServerXML) if err != nil { e.failedTests.Inc() return 0, err } // Close config at end of test. defer xmlF.Close() // Query XML for the AJP connector configuration. p, err := xmlquery.CreateStreamParser(xmlF, `//Connector[@protocol="AJP/1.3"]`) if err != nil { e.failedTests.Inc() return 0, err } // Variable to store found secrets. var foundSecrets []string // Pull the first connector match if possible. n, err := p.Read() // If EOF reached, no connectors are defined. if err == io.EOF { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("no AJP/1.3 connectors defined") } // Other erros are general. if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // Get the secret attribute from the connector. This only may be configured on older installs. secret := n.SelectAttr("secret") if secret != "" { foundSecrets = append(foundSecrets, secret) } // Get the required secret which is the newer attribute name. secret = n.SelectAttr("requiredSecret") if secret != "" { foundSecrets = append(foundSecrets, secret) } // If there are more than one secret, check if they both match. if len(foundSecrets) > 1 { if foundSecrets[0] != foundSecrets[1] { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("the AJP secrets do not match") } } // If no secrets were found, fail. if len(foundSecrets) == 0 { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("no AJP secrets found") } // Open the Apache HTTPD proxy configuration file. f, err := os.Open(app.config.HTTPDPKIProxyConf) if err != nil { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, err } // Close this config file at end of test. defer f.Close() // Create a new line scanner for the httpd configuration file. scanner := bufio.NewScanner(f) scanner.Split(bufio.ScanLines) // Parse regex for expected configuration. proxyRx := regexp.MustCompile(`\s+ProxyPassMatch ajp://localhost:8009 secret=(\w+)$`) // List of found secrets. var foundProxySecrets []string // Scan each line of the config for secrets. for scanner.Scan() { line := scanner.Text() // If line matches a secret, get the secret. if proxyRx.MatchString(line) { match := proxyRx.FindStringSubmatch(line) if len(match) == 2 { // Add secret to the list. foundProxySecrets = append(foundProxySecrets, match[1]) } } } // If no secrets found in HTTPD config, fail. if len(foundProxySecrets) == 0 { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("no AJP proxy secrets found") } // Check each found proxy secret against the tomcat secrets. for _, secret := range foundProxySecrets { foundMatch := false for _, xmlSecret := range foundSecrets { if secret == xmlSecret { foundMatch = true } } // If no match found, fail. if !foundMatch { // As this is a test, add to the tests failed counter. e.failedTests.Inc() return 0, fmt.Errorf("the AJP secrets configured do not match between tomcat and apache") } } // At this point, the test succeeded. return 1, nil }) // Info: Is this server the renewal master? e.NewMetric("renewal_master", "This server is the renewal master.", prometheus.GaugeValue, func() (float64, error) { // Get the FreeIPA config. config, err := e.apiConfig() if err != nil { return 0, err } // Get the configured renewal master. masterServer, _ := config.GetString("ca_renewal_master_server") // This is a renewal master if the configured hostname matches. if masterServer != app.config.Hostname { return 0, nil } return 1, nil }) // Info: Did the IPA CA sign the FreeIPA API certificate? e.NewMetric("ipa_ca_issued_cert", "The FreeIPA API was issued a certificate by the CA cert.", prometheus.GaugeValue, func() (float64, error) { // Get the CA certificate. caCert, err := e.caCert() if err != nil { return 0, err } // Get the FreeIPA API certificate. ipaCerts, err := e.ipaCerts() if err != nil { return 0, err } // Check each certificate returned by the API against the CA certificate. countSuccess := 0 for _, cert := range ipaCerts { // Ignore the CA certificates. if cert.IsCA { continue } // If the signature is signed by the CA certificate, add to the successes. err := cert.CheckSignatureFrom(caCert) if err == nil { countSuccess++ } } // If no successful signature checks, there are no certificates signed by the CA certificate. if countSuccess == 0 { return 0, fmt.Errorf("certificates are not issued by ca certificate") } // At this point, a certificate was found to be signed by the CA certificate. return 1, nil }) // Info: Is the FreeIPA API certificate in certmonger for autorenew? e.NewMetric("ipa_cert_auto_renew", "The FreeIPA API certificate is managed and set to auto renew.", prometheus.GaugeValue, func() (float64, error) { // Get certificates managed by cert monger. certsFound, err := e.certMongerCerts() if err != nil { return 0, err } // Get info about the httpd certificate. var httpSubject string autoRenew := false for _, cert := range certsFound { // If certificate storage path is in the httpd path, this is the httpd certificate. if strings.Contains(cert.KeyPairStorage, "httpd") { httpSubject = cert.Subject autoRenew = cert.Status == "MONITORING" && cert.AutoRenew && !cert.Stuck && cert.Track } } // Get the FreeIPA API certificates. ipaCerts, err := e.ipaCerts() if err != nil { return 0, err } // Check if the certificate currently returned by FreeIPA is the one found in cert monger. foundCerts := 0 for _, cert := range ipaCerts { // If is an CA certificate, skip. if cert.IsCA { continue } thisSubject := cert.Subject.String() // OpenSSL seems to escape the comma. thisSubject = strings.ReplaceAll(thisSubject, "\\,", ",") // If subject matches, this is the certificate in certmonger.. if thisSubject == httpSubject { foundCerts++ } } // If no certificates found, return error. if foundCerts == 0 { return 0, fmt.Errorf("unable to determine if http cert is auto renew") } // If not auto renew, return 0. if !autoRenew { return 0, nil } // At this point, the certificate was found and has been determined to be autorenew. return 1, nil }) // Info: The unix timestamp of the earliest FreeIPA API certificate expiry date. e.NewMetric("ipa_earliest_cert_expiry", "The earliest certificate expiry date for FreeIPA API.", prometheus.GaugeValue, func() (float64, error) { // Get FreeIPA API certificates. ipaCerts, err := e.ipaCerts() if err != nil { return 0, err } // Find the earliest expiry date. earliest := time.Time{} for _, cert := range ipaCerts { // If this is before the previously found earliest, update. if earliest.IsZero() || (cert.NotAfter.Before(earliest) && !cert.NotAfter.IsZero()) { earliest = cert.NotAfter } } // If the earliest date found is zero, we did not find any expiry dates. if earliest.IsZero() { return 0, fmt.Errorf("unable to find earliest cert") } // Return the earliest date in unix time format. return float64(earliest.Unix()), nil }) // Info: Did the IPA CA sign the LDAP certificate? e.NewMetric("ipa_ca_issued_ldap_cert", "The LDAP cert was issued a certificate by the CA cert.", prometheus.GaugeValue, func() (float64, error) { // Get the CA certificate. caCert, err := e.caCert() if err != nil { return 0, err } // Get the LDAP certificates. ldapCerts, err := e.ldapCerts() if err != nil { return 0, err } // Check each certificate. countSuccess := 0 for _, cert := range ldapCerts { // If this is a CA certificate, ignore. if cert.IsCA { continue } // If the signature was signed by the CA certificate, add to the count. err := cert.CheckSignatureFrom(caCert) if err == nil { countSuccess++ } } // If no successful signature checks, there are no certificates signed by the CA certificate. if countSuccess == 0 { return 0, fmt.Errorf("certificates are not issued by ca certificate") } // At this point, a certificate was found to be signed by the CA certificate. return 1, nil }) // Info: Is the LDAP certificate in certmonger for autorenew? e.NewMetric("ldap_cert_auto_renew", "The LDAP certificate is managed and set to auto renew.", prometheus.GaugeValue, func() (float64, error) { // Get the certificates managed by cert monger. certsFound, err := e.certMongerCerts() if err != nil { return 0, err } // Find info about the LDAP certificate in cert monger. var httpSubject string autoRenew := false for _, cert := range certsFound { // If the storage path is in the dirsrv folder, this is an LDAP certificate. if strings.Contains(cert.KeyPairStorage, "dirsrv") { httpSubject = cert.Subject autoRenew = cert.Status == "MONITORING" && cert.AutoRenew && !cert.Stuck && cert.Track } } // Get the LDAP certificates. ldapCerts, err := e.ldapCerts() if err != nil { return 0, err } // Check each LDAP certificate to see if it matches the one in cert monger. foundCerts := 0 for _, cert := range ldapCerts { // If CA certificate, ignore. if cert.IsCA { continue } thisSubject := cert.Subject.String() // OpenSSL seems to escape the comma. thisSubject = strings.ReplaceAll(thisSubject, "\\,", ",") // If this certificate matches the cert monger certificate. if thisSubject == httpSubject { foundCerts++ } } // If no certificates found, return error. if foundCerts == 0 { return 0, fmt.Errorf("unable to determine if LDAP cert is auto renew") } // If not auto renew, return 0. if !autoRenew { return 0, nil } // At this point, the certificate was found and has been determined to be autorenew. return 1, nil }) // Info: The unix timestamp of the earliest LDAP certificate expiry date. e.NewMetric("ldap_earliest_cert_expiry", "The earliest certificate expiry date for LDAP.", prometheus.GaugeValue, func() (float64, error) { // Get the LDAP certificates. ldapCerts, err := e.ldapCerts() if err != nil { return 0, err } // Find the earliest expiry date. earliest := time.Time{} for _, cert := range ldapCerts { // If this is before the previously found earliest, update. if earliest.IsZero() || (cert.NotAfter.Before(earliest) && !cert.NotAfter.IsZero()) { earliest = cert.NotAfter } } // If the earliest date found is zero, we did not find any expiry dates. if earliest.IsZero() { return 0, fmt.Errorf("unable to find earliest cert") } // Return the earliest date in unix time format. return float64(earliest.Unix()), nil }) } // Provide Promethues all descriptions of metrics exported. func (e *FreeIPAExporter) Describe(ch chan<- *prometheus.Desc) { for _, m := range e.metrics { ch <- m.Desc } ch <- e.up.Desc() ch <- e.totalScrapes.Desc() ch <- e.totalFailures.Desc() ch <- e.failedTests.Desc() } // Collects metrics exported and provide values to Prometheus. func (e *FreeIPAExporter) Collect(ch chan<- prometheus.Metric) { // Protect metrics from concurrent collects. e.mutex.Lock() defer e.mutex.Unlock() // Scrape FreeIPA metrics. up := e.scrape(ch) // Update the up status. e.up.Set(up) // If not up, count as a failed scrape. if up == 0 { e.totalFailures.Inc() } // Send basic metrics. ch <- e.up ch <- e.totalScrapes ch <- e.totalFailures ch <- e.failedTests } // Test FreeIPA and pull metrics. func (e *FreeIPAExporter) scrape(ch chan<- prometheus.Metric) float64 { // Reset the number of failed tests. e.failedTests.Set(0) // Increment the total number of scrapes. e.totalScrapes.Inc() // Attempt to connect to the FreeIPA API. err := e.connect() // If failure connecting, FreeIPA API is not up. if err != nil { log.Println("Error connecting to FreeIPA API:", err) return 0 } // Disconnect after we're done scrapping information. defer e.disconnect() // Update data for each metric. for _, m := range e.metrics { // Get the value of the metric. value, err := m.Value() // If an error occurred getting the value, log it for debug. if err != nil { log.Printf("Error retrieving value for metric %s: %s\n", m.Desc.String(), err) } // Update the value. ch <- prometheus.MustNewConstMetric(m.Desc, m.Type, value) } // The FreeIPA API server is up. return 1 }