diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/DefaultNotificationRuleProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/DefaultNotificationRuleProcessor.java index 62455117e0..d6cee80ebf 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/rule/DefaultNotificationRuleProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/DefaultNotificationRuleProcessor.java @@ -35,6 +35,7 @@ import org.thingsboard.server.common.data.notification.NotificationRequestStatus import org.thingsboard.server.common.data.notification.info.NotificationInfo; import org.thingsboard.server.common.data.notification.rule.NotificationRule; import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger; +import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger.DeduplicationStrategy; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerConfig; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; import org.thingsboard.server.common.data.plugin.ComponentLifecycleEvent; @@ -66,8 +67,8 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess private final NotificationDeduplicationService deduplicationService; private final PartitionService partitionService; private final RateLimitService rateLimitService; - @Autowired @Lazy - private NotificationCenter notificationCenter; + @Lazy + private final NotificationCenter notificationCenter; private final NotificationExecutorService notificationExecutor; private final Map triggerProcessors = new EnumMap<>(NotificationRuleTriggerType.class); @@ -82,14 +83,11 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess if (enabledRules.isEmpty()) { return; } - if (trigger.deduplicate()) { - enabledRules = new ArrayList<>(enabledRules); - enabledRules.removeIf(rule -> deduplicationService.alreadyProcessed(trigger, rule)); - } - final List rules = enabledRules; - for (NotificationRule rule : rules) { + + List rulesToProcess = filterNotificationRules(trigger, enabledRules); + for (NotificationRule rule : rulesToProcess) { try { - processNotificationRule(rule, trigger); + processNotificationRule(rule, trigger, DeduplicationStrategy.ONLY_MATCHING.equals(trigger.getDeduplicationStrategy())); } catch (Throwable e) { log.error("Failed to process notification rule {} for trigger type {} with trigger object {}", rule.getId(), rule.getTriggerType(), trigger, e); } @@ -100,7 +98,20 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess }); } - private void processNotificationRule(NotificationRule rule, NotificationRuleTrigger trigger) { + private List filterNotificationRules(NotificationRuleTrigger trigger, List enabledRules) { + List rulesToProcess = new ArrayList<>(enabledRules); + rulesToProcess.removeIf(rule -> switch (trigger.getDeduplicationStrategy()) { + case ONLY_MATCHING -> { + boolean matched = matchesFilter(trigger, rule.getTriggerConfig()); + yield !matched || deduplicationService.alreadyProcessed(trigger, rule); + } + case ALL -> deduplicationService.alreadyProcessed(trigger, rule); + default -> false; + }); + return rulesToProcess; + } + + private void processNotificationRule(NotificationRule rule, NotificationRuleTrigger trigger, boolean alreadyMatched) { NotificationRuleTriggerConfig triggerConfig = rule.getTriggerConfig(); log.debug("Processing notification rule '{}' for trigger type {}", rule.getName(), rule.getTriggerType()); @@ -114,7 +125,7 @@ public class DefaultNotificationRuleProcessor implements NotificationRuleProcess return; } - if (matchesFilter(trigger, triggerConfig)) { + if (alreadyMatched || matchesFilter(trigger, triggerConfig)) { if (!rateLimitService.checkRateLimit(LimitedApi.NOTIFICATION_REQUESTS_PER_RULE, rule.getTenantId(), rule.getId())) { log.debug("[{}] Rate limit for notification requests per rule was exceeded (rule '{}')", rule.getTenantId(), rule.getName()); return; diff --git a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/ResourcesShortageTriggerProcessor.java b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/ResourcesShortageTriggerProcessor.java index aefb628d2d..09c8d76eca 100644 --- a/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/ResourcesShortageTriggerProcessor.java +++ b/application/src/main/java/org/thingsboard/server/service/notification/rule/trigger/ResourcesShortageTriggerProcessor.java @@ -39,7 +39,12 @@ public class ResourcesShortageTriggerProcessor implements NotificationRuleTrigge @Override public RuleOriginatedNotificationInfo constructNotificationInfo(ResourcesShortageTrigger trigger) { - return ResourcesShortageNotificationInfo.builder().resource(trigger.getResource().name()).usage(trigger.getUsage()).build(); + return ResourcesShortageNotificationInfo.builder() + .resource(trigger.getResource().name()) + .usage(trigger.getUsage()) + .serviceId(trigger.getServiceId()) + .serviceType(trigger.getServiceType()) + .build(); } @Override diff --git a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java index adc87957d3..a38c2721c1 100644 --- a/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java +++ b/application/src/main/java/org/thingsboard/server/service/system/DefaultSystemInfoService.java @@ -185,9 +185,14 @@ public class DefaultSystemInfoService extends TbApplicationEventListener clusterSystemData = getSystemData(serviceInfoProvider.getServiceInfo()); clusterSystemData.forEach(data -> { - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.CPU).usage(data.getCpuUsage()).build()); - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.RAM).usage(data.getMemoryUsage()).build()); - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.STORAGE).usage(data.getDiscUsage()).build()); + Arrays.stream(Resource.values()).forEach(resource -> { + notificationRuleProcessor.process(ResourcesShortageTrigger.builder() + .resource(resource) + .serviceId(data.getServiceId()) + .serviceType(data.getServiceType()) + .usage(extractResourceUsage(data, resource)) + .build()); + }); }); BasicTsKvEntry clusterDataKv = new BasicTsKvEntry(ts, new JsonDataEntry("clusterSystemData", JacksonUtil.toString(clusterSystemData))); doSave(Arrays.asList(new BasicTsKvEntry(ts, new BooleanDataEntry("clusterMode", true)), clusterDataKv)); @@ -200,17 +205,17 @@ public class DefaultSystemInfoService extends TbApplicationEventListener { long value = (long) v; tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("cpuUsage", value))); - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.CPU).usage(value).build()); + notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.CPU).usage(value).serviceId(serviceInfoProvider.getServiceId()).serviceType(serviceInfoProvider.getServiceType()).build()); }); getMemoryUsage().ifPresent(v -> { long value = (long) v; tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("memoryUsage", value))); - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.RAM).usage(value).build()); + notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.RAM).usage(value).serviceId(serviceInfoProvider.getServiceId()).serviceType(serviceInfoProvider.getServiceType()).build()); }); getDiscSpaceUsage().ifPresent(v -> { long value = (long) v; tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("discUsage", value))); - notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.STORAGE).usage(value).build()); + notificationRuleProcessor.process(ResourcesShortageTrigger.builder().resource(Resource.STORAGE).usage(value).serviceId(serviceInfoProvider.getServiceId()).serviceType(serviceInfoProvider.getServiceType()).build()); }); getCpuCount().ifPresent(v -> tsList.add(new BasicTsKvEntry(ts, new LongDataEntry("cpuCount", (long) v)))); @@ -258,6 +263,14 @@ public class DefaultSystemInfoService extends TbApplicationEventListener info.getCpuUsage(); + case RAM -> info.getMemoryUsage(); + case STORAGE -> info.getDiscUsage(); + }; + } + @PreDestroy private void destroy() { if (scheduler != null) { diff --git a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java index 282e959fcd..bab70ea505 100644 --- a/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java +++ b/application/src/test/java/org/thingsboard/server/service/notification/NotificationRuleApiTest.java @@ -825,6 +825,8 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { notificationRuleProcessor.process(ResourcesShortageTrigger.builder() .resource(Resource.RAM) .usage(15L) + .serviceType("serviceType") + .serviceId("serviceId") .build()); TimeUnit.MILLISECONDS.sleep(300); } @@ -837,10 +839,48 @@ public class NotificationRuleApiTest extends AbstractNotificationApiTest { notificationRuleProcessor.process(ResourcesShortageTrigger.builder() .resource(Resource.RAM) .usage(5L) + .serviceType("serviceType") + .serviceId("serviceId") .build()); await("").atMost(5, TimeUnit.SECONDS).untilAsserted(() -> assertThat(getMyNotifications(false, 100)).size().isOne()); } + @Test + public void testNotificationsResourcesShortage_whenThresholdChangeToMatchingFilter_thenSendNotification() throws Exception { + loginSysAdmin(); + ResourcesShortageNotificationRuleTriggerConfig triggerConfig = ResourcesShortageNotificationRuleTriggerConfig.builder() + .ramThreshold(1f) + .cpuThreshold(1f) + .storageThreshold(1f) + .build(); + NotificationRule rule = createNotificationRule(triggerConfig, "Warning: ${resource} shortage", "${resource} shortage", createNotificationTarget(tenantAdminUserId).getId()); + loginTenantAdmin(); + + Method method = DefaultSystemInfoService.class.getDeclaredMethod("saveCurrentMonolithSystemInfo"); + method.setAccessible(true); + method.invoke(systemInfoService); + + TimeUnit.SECONDS.sleep(5); + assertThat(getMyNotifications(false, 100)).size().isZero(); + + loginSysAdmin(); + triggerConfig = ResourcesShortageNotificationRuleTriggerConfig.builder() + .ramThreshold(0.01f) + .cpuThreshold(1f) + .storageThreshold(1f) + .build(); + rule.setTriggerConfig(triggerConfig); + saveNotificationRule(rule); + loginTenantAdmin(); + + method.invoke(systemInfoService); + + await().atMost(10, TimeUnit.SECONDS).until(() -> getMyNotifications(false, 100).size() == 1); + Notification notification = getMyNotifications(false, 100).get(0); + assertThat(notification.getSubject()).isEqualTo("Warning: RAM shortage"); + assertThat(notification.getText()).isEqualTo("RAM shortage"); + } + @Test public void testNotificationRuleDisabling() throws Exception { EntityActionNotificationRuleTriggerConfig triggerConfig = new EntityActionNotificationRuleTriggerConfig(); diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java index c979fbffd1..afbad6e3a2 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/SystemInfoData.java @@ -20,6 +20,7 @@ import lombok.Data; @Data public class SystemInfoData { + @Schema(description = "Service Id.") private String serviceId; @Schema(description = "Service type.") diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/ResourcesShortageNotificationInfo.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/ResourcesShortageNotificationInfo.java index 24cb21febd..c05a10ef8f 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/ResourcesShortageNotificationInfo.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/info/ResourcesShortageNotificationInfo.java @@ -30,12 +30,16 @@ public class ResourcesShortageNotificationInfo implements RuleOriginatedNotifica private String resource; private Long usage; + private String serviceId; + private String serviceType; @Override public Map getTemplateData() { return Map.of( "resource", resource, - "usage", String.valueOf(usage) + "usage", String.valueOf(usage), + "serviceId", serviceId, + "serviceType", serviceType ); } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeCommunicationFailureTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeCommunicationFailureTrigger.java index 4124eb04f8..5672b2c98c 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeCommunicationFailureTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeCommunicationFailureTrigger.java @@ -23,12 +23,16 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import java.io.Serial; import java.util.concurrent.TimeUnit; @Data @Builder public class EdgeCommunicationFailureTrigger implements NotificationRuleTrigger { + @Serial + private static final long serialVersionUID = 2918443863787603524L; + private final TenantId tenantId; private final CustomerId customerId; private final EdgeId edgeId; @@ -37,8 +41,8 @@ public class EdgeCommunicationFailureTrigger implements NotificationRuleTrigger private final String error; @Override - public boolean deduplicate() { - return true; + public DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.ALL; } @Override @@ -60,4 +64,5 @@ public class EdgeCommunicationFailureTrigger implements NotificationRuleTrigger public EntityId getOriginatorEntityId() { return edgeId; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeConnectionTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeConnectionTrigger.java index fc3b69e697..0da465ec09 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeConnectionTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/EdgeConnectionTrigger.java @@ -23,12 +23,16 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import java.io.Serial; import java.util.concurrent.TimeUnit; @Data @Builder public class EdgeConnectionTrigger implements NotificationRuleTrigger { + @Serial + private static final long serialVersionUID = -261939829962721957L; + private final TenantId tenantId; private final CustomerId customerId; private final EdgeId edgeId; @@ -36,8 +40,8 @@ public class EdgeConnectionTrigger implements NotificationRuleTrigger { private final String edgeName; @Override - public boolean deduplicate() { - return true; + public DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.ALL; } @Override @@ -59,4 +63,5 @@ public class EdgeConnectionTrigger implements NotificationRuleTrigger { public EntityId getOriginatorEntityId() { return edgeId; } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NewPlatformVersionTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NewPlatformVersionTrigger.java index 204ae15e57..50ee0768d9 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NewPlatformVersionTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NewPlatformVersionTrigger.java @@ -22,10 +22,15 @@ import org.thingsboard.server.common.data.id.EntityId; import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import java.io.Serial; + @Data @Builder public class NewPlatformVersionTrigger implements NotificationRuleTrigger { + @Serial + private static final long serialVersionUID = 3298785969736390092L; + private final UpdateMessage updateInfo; @Override @@ -45,8 +50,8 @@ public class NewPlatformVersionTrigger implements NotificationRuleTrigger { @Override - public boolean deduplicate() { - return true; + public DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.ALL; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTrigger.java index 31940d6ac8..f6cf398b44 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/NotificationRuleTrigger.java @@ -29,9 +29,8 @@ public interface NotificationRuleTrigger extends Serializable { EntityId getOriginatorEntityId(); - - default boolean deduplicate() { - return false; + default DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.NONE; } default String getDeduplicationKey() { @@ -43,4 +42,10 @@ public interface NotificationRuleTrigger extends Serializable { return 0; } + enum DeduplicationStrategy { + NONE, + ALL, + ONLY_MATCHING + } + } diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/RateLimitsTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/RateLimitsTrigger.java index 39d570a9e0..37e984c6f5 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/RateLimitsTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/RateLimitsTrigger.java @@ -22,12 +22,16 @@ import org.thingsboard.server.common.data.id.TenantId; import org.thingsboard.server.common.data.limit.LimitedApi; import org.thingsboard.server.common.data.notification.rule.trigger.config.NotificationRuleTriggerType; +import java.io.Serial; import java.util.concurrent.TimeUnit; @Data @Builder public class RateLimitsTrigger implements NotificationRuleTrigger { + @Serial + private static final long serialVersionUID = -4423112145409424886L; + private final TenantId tenantId; private final LimitedApi api; private final EntityId limitLevel; @@ -45,8 +49,8 @@ public class RateLimitsTrigger implements NotificationRuleTrigger { @Override - public boolean deduplicate() { - return true; + public DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.ALL; } @Override diff --git a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/ResourcesShortageTrigger.java b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/ResourcesShortageTrigger.java index f12c80d5db..be2485c959 100644 --- a/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/ResourcesShortageTrigger.java +++ b/common/data/src/main/java/org/thingsboard/server/common/data/notification/rule/trigger/ResourcesShortageTrigger.java @@ -33,6 +33,8 @@ public class ResourcesShortageTrigger implements NotificationRuleTrigger { private Resource resource; private Long usage; + private String serviceId; + private String serviceType; @Override public TenantId getTenantId() { @@ -45,13 +47,13 @@ public class ResourcesShortageTrigger implements NotificationRuleTrigger { } @Override - public boolean deduplicate() { - return true; + public DeduplicationStrategy getDeduplicationStrategy() { + return DeduplicationStrategy.ONLY_MATCHING; } @Override public String getDeduplicationKey() { - return resource.name(); + return String.join(":", resource.name(), serviceId, serviceType); } @Override diff --git a/common/queue/src/main/java/org/thingsboard/server/queue/notification/RemoteNotificationRuleProcessor.java b/common/queue/src/main/java/org/thingsboard/server/queue/notification/RemoteNotificationRuleProcessor.java index 194f0ce962..a410ac5d04 100644 --- a/common/queue/src/main/java/org/thingsboard/server/queue/notification/RemoteNotificationRuleProcessor.java +++ b/common/queue/src/main/java/org/thingsboard/server/queue/notification/RemoteNotificationRuleProcessor.java @@ -22,6 +22,7 @@ import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean import org.springframework.stereotype.Service; import org.thingsboard.server.common.data.JavaSerDesUtil; import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger; +import org.thingsboard.server.common.data.notification.rule.trigger.NotificationRuleTrigger.DeduplicationStrategy; import org.thingsboard.server.common.msg.notification.NotificationRuleProcessor; import org.thingsboard.server.common.msg.queue.ServiceType; import org.thingsboard.server.common.msg.queue.TopicPartitionInfo; @@ -47,7 +48,7 @@ public class RemoteNotificationRuleProcessor implements NotificationRuleProcesso @Override public void process(NotificationRuleTrigger trigger) { try { - if (trigger.deduplicate() && deduplicationService.alreadyProcessed(trigger)) { + if (!DeduplicationStrategy.NONE.equals(trigger.getDeduplicationStrategy()) && deduplicationService.alreadyProcessed(trigger)) { return; } diff --git a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java index efd69a4e61..7cee3d04ae 100644 --- a/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java +++ b/dao/src/main/java/org/thingsboard/server/dao/notification/DefaultNotifications.java @@ -376,7 +376,7 @@ public class DefaultNotifications { public static final DefaultNotification resourcesShortage = DefaultNotification.builder() .name("Resources shortage notification") .type(NotificationType.RESOURCES_SHORTAGE) - .subject("Warning: ${resource} shortage") + .subject("Warning: ${resource} shortage for ${serviceId}") .text("${resource} usage is at ${usage}%.") .icon("warning") .rule(DefaultRule.builder() diff --git a/ui-ngx/src/assets/help/en_US/notification/resources_shortage.md b/ui-ngx/src/assets/help/en_US/notification/resources_shortage.md index 9b469c0f85..6ea03a514c 100644 --- a/ui-ngx/src/assets/help/en_US/notification/resources_shortage.md +++ b/ui-ngx/src/assets/help/en_US/notification/resources_shortage.md @@ -9,8 +9,10 @@ See the available types and parameters below: Available template parameters: -* `resource` - the resource name; -* `usage` - the resource usage value; +* `resource` - the resource name (e.g., "CPU", "RAM", "STORAGE"); +* `usage` - the current usage value of the resource; +* `serviceId` - the service id (convenient in cluster setup); +* `serviceType` - the service type (convenient in cluster setup); Parameter names must be wrapped using `${...}`. For example: `${resource}`. You may also modify the value of the parameter with one of the suffixes: